diff --git a/.helix/languages.toml b/.helix/languages.toml deleted file mode 100644 index 529cdcf3..00000000 --- a/.helix/languages.toml +++ /dev/null @@ -1,19 +0,0 @@ -[[language]] -auto-format = true -file-types = [ "zig" ] -language-servers = [ "zls" ] -name = "zig" -roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] -scope = "source.zig" - -[language.indent] -tab-width = 4 -unit = " " - -[language-server.zls] -command = "sh" -args = [ - "-c", - "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", - "zls" -] diff --git a/README.md b/README.md index d9e68b81..216fc165 100644 --- a/README.md +++ b/README.md @@ -162,29 +162,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - meshes: []const *const zml.Mesh, - ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .meshes = meshes, - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -195,8 +172,46 @@ const Mnist = struct { return x.argMax(0).indices.convert(.u8); } }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const path = ...; + + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + const model: Mnist = .init(store.view()); + + const input: zml.Tensor = .init(.{ 28, 28 }, .u8); + var exe = try platform.compile(allocator, io, model, .forward, .{input}, .{}); + defer exe.deinit(); + + var weights = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &weights); + try loader.start(&model, &weights, &store, .none); + try loader.wait(); // logs "Loaded weights" + + // ... call `exe` on the weights and an image +} ``` +The first `start` waits for whatever calibration is left. +[Loading checkpoints](./docs/learn/loader.md) gives the full recipe, with +remote checkpoints, placement, lifetimes and errors. + For a full walkthrough, see: - [Getting Started](./docs/tutorials/getting_started.md) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index eb2b2ab2..434ecfa9 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -3,9 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "retain_runfiles_init", - srcs = ["runfiles_init.c"], - # Retain the constructor even when its object is in a static archive. - # Mach-O adds an underscore to the exported C symbol. + # Mach-O adds an underscore to the exported Zig symbol. linkopts = select({ "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 67513227..2ae3324a 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,26 +1,35 @@ const std = @import("std"); +const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; -} +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c deleted file mode 100644 index 770a4b91..00000000 --- a/bazel/runfiles_init.c +++ /dev/null @@ -1,15 +0,0 @@ -extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); - -// Preserve the existing startup ABI. Registration happens in C; -// runfiles discovery and initialization remain in Zig. -// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. -// The original Zig initializer already made this assumption to read argv[0]. -// -// Let C emit the platform's constructor registration. Zig's incremental ELF -// linker does not yet handle .init_array's section type, flags, and size correctly. -// TODO: Revert this workaround once Zig issue #37453 is fixed: -// https://codeberg.org/ziglang/zig/issues/37453 -__attribute__((constructor)) -void _bazel_runfiles_init(int argc, char **argv, char **envp) { - _bazel_runfiles_init_impl(argc, argv, envp); -} diff --git a/devenv.lock b/devenv.lock index 6c3be332..bb3ef660 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1793199252, - "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", + "lastModified": 1779636362, + "narHash": "sha256-Q/hy/ybvbe91fjA1MbulUwPjJH9yKaFZnSaTkwLE8S0=", "owner": "cachix", "repo": "devenv", - "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", + "rev": "9d6e68dac81fab1f7f2ec6784220151b4fba4b59", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1793043970, - "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", + "lastModified": 1778507786, + "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", + "rev": "8f24a228a782e24576b155d1e39f0d914b380691", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1792812153, - "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", + "lastModified": 1778274207, + "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", + "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", "type": "github" }, "original": { diff --git a/docs/README.md b/docs/README.md index 03369120..3a356509 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Learn more... - [ZML Concepts](./learn/concepts.md) : **Tensors, Models, Executables, etc. explained** +- [Loading checkpoints](./learn/loader.md) : **The loading recipe, placement, programs, lifetimes and errors** ## Contribute diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 3d246f11..d2c63bc1 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -160,7 +160,6 @@ bazel build //my_project:torch2zml ./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors -debug(zml/safetensors): Added new metadata key=format with value=pt info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` @@ -190,7 +189,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); + var model_store: zml.io.TensorStore = .init(allocator, &model_registry, .{}); defer model_store.deinit(); } ``` @@ -206,17 +205,17 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, .replicated, .{}), .d, ), .gate_proj = .init( mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), + mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, .replicated, .{}), .d, ), .down_proj = .init( mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), - mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), + mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, .replicated, .{}), .dout, ), }; @@ -235,15 +234,16 @@ Python script. Alternatively, Huggingface's web-interface allows to peek into ## Loading an individual layer Currently, the layer is initialized but the weights are still on the disk. -ZML provides a convenience to automatically load weights from an initialized model: +ZML provides a convenience to load weights from an initialized model in a test: ```zig pub fn main(init: std.process.Init) !void { ... var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); + defer platform.deinit(allocator, io); + var mlp_weights = try zml.testing.load(allocator, io, platform, &mlp, &model_store); + defer zml.mem.deinitBufferized(allocator, &mlp_weights); } ``` @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); + var activations_store: zml.io.TensorStore = .init(allocator, &activations_registry, .{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index cc11ffbb..9f18b4a7 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -17,7 +17,7 @@ access token to log into your account on Huggingface, directly from `bazel`, in * the token looks something like `hf_abCdEfGhijKlM`. * store the token on your machine (replace the placeholder with your actual token): -You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use +You can use the `HF_TOKEN` environment variable to store the token or use its standard location: ``` mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" @@ -29,6 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +# `hf auth login` command, or the `HF_TOKEN` environment variable. bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/learn/loader.md b/docs/learn/loader.md new file mode 100644 index 00000000..d97b363c --- /dev/null +++ b/docs/learn/loader.md @@ -0,0 +1,210 @@ +# Loading checkpoints + +`zml.io.Loader` reads a model's weights from its checkpoint into device +buffers. Model code binds each tensor to checkpoint bytes through a +`TensorStore` view; the loader fills the buffers, assembles the tensors made +of several checkpoint tensors, and runs the programs that compute others. +The doc comments of `zml/io/loader.zig`, `TensorStore.zig`, `Source.zig` and +`vfs/vfs.zig` are the contract; the internals are in the `//!` header of +each file `zml/io.zig` lists. + +## The recipe + +```zig +// Local paths, and hf://, s3://, gs:// and https:// through `Remotes`, +// configured from the environment (tokens, credentials, proxies). +var vfs: zml.io.VFS = try .init(allocator, init.io); +defer vfs.deinit(); +var remotes: zml.io.VFS.Remotes = undefined; // initialized in place +try remotes.init(allocator, &vfs, init.environ_map); +defer remotes.deinit(); +const io = vfs.io(); // for the platform, the registry and the loader + +const platform: *zml.Platform = try .auto(allocator, io, .{}); +defer platform.deinit(allocator, io); + +// Returns at once: calibration runs while the model is built and compiled. +var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); +defer loader.deinit(); + +// The checkpoint's headers. +var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); +defer registry.deinit(); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); +defer store.deinit(); + +const model: Model = .init(store.view()); // binds every tensor +// ... compile the model ... + +var buffers = try zml.mem.bufferize(allocator, &model); +defer zml.mem.deinitBufferized(allocator, &buffers); +try loader.start(&model, &buffers, &store, .none); +try loader.wait(); // logs "Loaded weights [bytes, time, rate]" +``` + +`.checkpoint` lets the VFS backend serving the path size the reads, and the +loader place its pinned memory for how a local checkpoint reads; without it, +or without a VFS, reads are sized for local files. `start` adds its own items +to the progress node it is given. Several loads may be started before one +`wait`, from one store or several. A second checkpoint, such as a drafter's, +joins the registry under a prefix before the store takes it: +`registry.addRepo(io, drafter_repo, "drafter")`, then +`store.view().withPrefix("drafter")` reaches its tensors. + +## Placement + +A tensor is placed by its shape. `View.createTensor(subkey, tags, .tp, +partitioning)` places the checkpoint tensor on the mesh named `.tp` among the +store's `meshes` (`View.mesh(.tp)` returns it), each tag of `partitioning` +naming the mesh axis that splits it; `createReplicatedTensor` puts it whole +on every device. It lands in default device memory, unless `View.setMemory`, +called after binding, names a host kind (`TensorStore.Memory`): +`.host_pinned` gives buffers in pinned host memory, and `.host_mapped` gives +device buffers that are views of pinned host memory, which kernels read in +place. The second suits a table too large for device memory, of which each +step reads a few rows; the memory goes with the last view. XLA counts those +views against device memory: give their size on one device, +`view.hostMappedBytes()`, back through the compile option +`extra_device_memory_bytes`. + +```zig +const tp = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); +var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tp} }); +const view = store.view(); +const w = view.createTensor("w", .{ .out, .in }, .tp, .{ .out = .model }); +const b = view.createReplicatedTensor("b", .{.out}); +const rows = view.createReplicatedTensor("rows", .{ .row, .d }); +view.setMemory(rows, .host_mapped); +``` + +`View.get(subkey)` returns the registry's entry for a key, or null, and +`count()` the number of keys under the view's prefix. Binding cannot fail: +`createTensor` and `source` panic on a missing key, `maybeCreateTensor` and +`maybeSource` return null. `TensorStore.Options.quantization` +(`Quantization.Target.of(platform)`) is required to bind block-FP8 weights. + +## Composite tensors + +A composite tensor is one destination filled from several checkpoint byte +ranges, every byte no range covers set to zero, with no program and no +buffer per source. It is a `TensorStore.Source`: a checkpoint tensor +(`View.source(subkey, tags)`) and `zml.Tensor`'s shape ops over it, with the +same names, tags and arguments: `slice`, `split`, `chunkExact`, `splitAxis`, +`merge`, `reshape`, `transpose`, `pad` (with zeros), `broad`, `bitCast`, +`concatenate` and `stack`. Placement comes last, after every op: +`withPartitioning(mesh, partitioning)`, keyed by the source's final tags, or +`as(shape)`, a shape built elsewhere with the source's element count and +dtype. Both return a `Source.Placed`, which has no shape ops (its `.shape` +is the placed shape) and drops its placement with `unplaced()`; its +`bind()` makes the model tensor. + +```zig +const Source = zml.io.TensorStore.Source; +// a, b and c end to end along the rows, padded to 16 rows, row-sharded. +const abc = Source.concatenate(&.{ view.source("a", .{ .row, .d }), view.source("b", .{ .row, .d }), view.source("c", .{ .row, .d }) }, .row); +const table = abc.pad(0, .{ .row = Source.Pad{ .high = 16 - abc.dim(.row) } }) + .withPartitioning(.tp, .{ .row = .model }).bind(); +// q, k and v cut in one group per device and interleaved, so that each +// device holds its rows of every projection. +var members: [3]Source = undefined; +for (&members, [_][]const u8{ "q.weight", "k.weight", "v.weight" }) |*m, key| { + m.* = view.source(key, .{ .dout, .d }).splitAxis(.dout, .{ .group = devices, .r = -1 }); +} +const qkv = Source.concatenate(&members, .r).merge(.{ .dout = .{ .group, .r } }).withPartitioning(.tp, .{ .dout = .model }).bind(); +``` + +Sources live in their store's arena, and sources of two stores do not mix. +The loader only moves bytes: large runs are cheap, runs under 4 KiB cost +host copies. For a layout of many small runs, a program over sources read +in large runs may load faster, at the cost of a compile and staging memory +(measured in the header of `zml/io/direct_loader.zig`). + +## Programs + +What computes rather than moves bytes is a program: a dtype conversion, a +dequantization, a global scale. A program is a value of its own type, plain +data (checked at compile time), with a `forward` method; its type name +names the compiled module in traces and logs. + +```zig +const Dequantize = struct { + block_size: i64, + + pub fn forward(self: Dequantize, output: zml.Shape, inputs: []const zml.Tensor) zml.Tensor { + // One scale per row and block of `block_size` columns. + const blocks = inputs[0].splitAxis(.in, .{ .block = -1, .k = self.block_size }); + const y = blocks.mul(inputs[1].broad(blocks.shape())).merge(.{ .in = .{ .block, .k } }); + return y.convert(output.dtype()).withPartitioning(output._sharding.mesh.?, output._sharding.partition); + } +}; +const w = view.source("w", .{ .out, .in }); +const s = view.source("w_scale", .{ .out, .block }); +// Each device reads its rows of `w`, and all of `s`. +const placed = w.shape().withPartitioning(tp, .{ .out = .model }); +const weight = Source.bindProgram(&.{ w.as(placed), s.withPartitioning(.replicated, .{}) }, placed.withDtype(.bf16), Dequantize{ .block_size = 128 }); +``` + +`bindProgram(inputs, output, program)` binds a tensor of `output` that +`program.forward(output, inputs)` computes during the load and returns +placed as `output` is. Each input is a `Source.Placed` and lands placed by +its own shape, so every device reads only its slice; a whole checkpoint +tensor, replicated, is +`view.source(key, null).withPartitioning(.replicated, .{})`. `start` +starts one compile for all the tensors of an equal program (same type, +equal value) over the same placed shapes. `wait` lands the inputs in groups +of at most `Loader.Options.program_group_bytes` of outputs (8 GiB), runs +the programs on the calling task and frees the inputs. While other groups +are pending, a group waits until the devices have room for it; once none +is pending, it goes anyway, and a group that does not fit a device fails +`wait` with the device's error. Before `wait` returns, the load gives the +device memory it freed, inputs and temporaries, back to the devices; XLA's +VMM allocator, the default on CUDA and ROCm, would otherwise keep counting +it as in use. A program's output lands in default device memory: +`setMemory` panics on it. + +## Lifetime and failure + +| Handle | Borrows | Until | +| --- | --- | --- | +| `Loader` | the platform; `io`, and the VFS behind it, through which it may keep its source files open | its `deinit` | +| `Loader.start` | the store, the buffers and the progress node | `wait` or `cancel` returns | +| `TensorStore` | the meshes; it owns the registry's entries, the bindings, sources and programs | its `deinit` | +| `View`, `Source`, `Source.Placed` | their store, which must not move | their last use | +| `VFS.Remotes` | the VFS; it must not move | its `deinit`, once every file opened through it is closed | +| Loaded buffers | the platform | `deinitBufferized` | + +`init` reads its options and `start` the model during the call only. The +loader copies each file's path, so nothing of the store is kept past `wait`. +The loaded buffers are the caller's and outlive the loader, which holds its +pinned memory and workers until `deinit`: the recipe's deferred `deinit` +keeps them until the function returns, which suits a program that does +little after the load; one that keeps running frees the loader right after +`wait`, or loads in a block of its own. Deferred cleanup in the recipe's +order is right; if code between `start` and `wait` can fail, +`errdefer loader.cancel();` after `start`. The loader is a value. Start and +wait on one task: the backend reads and transfers concurrently. + +Every error of `start` or `wait` fails the loader: everything started is +canceled, and every later call returns the first error. `cancel` gives up on +what is started without running its programs. +Whatever failed, each buffer is empty or holds shards whose contents are +unspecified: read none, and free the tree with `zml.mem.deinitBufferized`. +That holds for the buffers `bufferize` made, which `start` empties again. + +## Errors + +`Loader.init` returns `InvalidOptions`, logged with the option it refuses, +or `OutOfMemory` when it cannot copy a local checkpoint's path; an error of +the backend's set-up is returned by the first `start`. +`start` and `wait` return the error where it was detected, under its own +name, and that is where it is logged, once, at err, with what explains it; +a plain allocation failure and `Canceled` are returned unlogged. + +A model whose bindings are wrong is `InvalidBinding`; the log names the +tensor (for a program's output, the program), the key its source reads and +the rule it breaks (`Loader.start`, `Loader.wait`). + +The other errors a caller can act on are listed in `Loader`'s doc comment: +a missing or denied file, a file that ends inside a tensor, host or device +memory that runs out, an `io` without concurrency, and `Canceled`. PJRT, +compile and remote failures pass through as their source names them. diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 312c9afc..fc394979 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -6,7 +6,8 @@ - clone ZML to work directly within the prepared example folder - add Zig code to implement our model - add some Bazel to integrate our code with ZML -- no weights files or anything external is required for this example +- no weights files or anything external is required, until the last step + loads the weights from a checkpoint The reason we're doing our exercise in the `examples` folder is because it's especially prepared for new ZML projects. It contains everything needed for ZML @@ -58,10 +59,12 @@ _Module_. For compilation, only the _Tensors_ are required. No actual tensor dat is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything -that is required to make _Tensors_ and later materialize them to _Buffers_. In our -example though, we won't even use a _TensorStore_ as it's optional. We will directly -create tensors using their shape and data type. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ holds the shapes +of a checkpoint's tensors, so that we can make _Tensors_, and the _Loader_ later +materializes them to _Buffers_. At first we won't use them: we will directly +create tensors using their shape and data type, and fill their buffers from +slices. The last step, [Loading weights](#loading-weights), reads them from a +checkpoint instead. After compilation is done, we get what we call an _Executable_. From this _Executable_ we can create _Args_ and _Results_, two structs that will store respectively the inputs @@ -175,9 +178,9 @@ var executable = try platform.compile(allocator, io, layer, .forward, .{input}, defer executable.deinit(); ``` -You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, -and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. -For this simple example, we just replicate the tensors across all devices for simplicity. +ZML supports sharding tensors across multiple devices, and `compile()` reads how +each tensor is sharded from its shape. A tensor created from a shape alone, as +here, is replicated on every device. ### Creating the "bufferized" Model @@ -191,8 +194,8 @@ provides ways to have extra-low loading times). const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); @@ -213,7 +216,7 @@ To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs @@ -227,7 +230,7 @@ defer results.deinit(allocator); args.set(.{ layer_buffers, input_buffer }); // call our executable -executable.call(io, args, &results, .{}); +try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -372,15 +375,15 @@ pub fn main(init: std.process.Init) !void { const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); // create the Args and Results structs @@ -394,7 +397,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ layer_buffers, input_buffer }); // call our executable - executable.call(io, args, &results, .{}); + try executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -411,10 +414,168 @@ pub fn main(init: std.process.Init) !void { } ``` +## Loading weights + +Real models read their weights from a checkpoint. Let's write one for our +layer, a `.safetensors` file holding `weight` and `bias`, with a few lines of +Python: + +```python +import json, struct + +weight = struct.pack("<3e", 1.0, 2.0, 3.0) # three f16 values +bias = struct.pack("<3e", 1.0, 1.0, 1.0) +header = json.dumps({ + "weight": {"dtype": "F16", "shape": [3], "data_offsets": [0, 6]}, + "bias": {"dtype": "F16", "shape": [3], "data_offsets": [6, 12]}, +}).encode() +with open("/tmp/layer.safetensors", "wb") as f: + f.write(struct.pack(" .cta_1, - .two => .cta_2, - }; - } -}; -pub const MBarrierScope = nvvm.MBarrierScope; -pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; - -/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. -pub fn ctaCount(group: CtaGroup) u8 { - return @backingInt(group); -} - -pub const Tcgen05SmemSwizzle = enum(u3) { - none = 0, - swizzle_128b_atom_32b = 1, - swizzle_128b = 2, - swizzle_64b = 4, - swizzle_32b = 6, -}; - -/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. -pub const Tcgen05SmemDesc = struct { - pub const Options = struct { - leading_byte_offset: u32 = 0, - stride_byte_offset: u32 = 0, - version: u2 = 1, - base_offset: u3 = 0, - layout: Tcgen05SmemSwizzle = .none, - }; - - pub fn build(b: *Builder, start_address: Value, opts: Options) Value { - std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); - std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); - const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | - (@as(u64, opts.stride_byte_offset >> 4) << 32) | - (@as(u64, opts.version) << 46) | - (@as(u64, opts.base_offset) << 49) | - (@as(u64, @backingInt(opts.layout)) << 61); - const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); - return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); - } -}; - -/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. -pub const Tcgen05InstrDesc = struct { - pub const Options = struct { - a_dtype: DType, - b_dtype: DType, - c_dtype: DType = .f32, - m_dim: u16, - n_dim: u16, - a_major: OperandMajorMode = .k, - b_major: OperandMajorMode = .k, - a_negate: bool = false, - b_negate: bool = false, - }; - - pub fn build(comptime opts: Options) u32 { - comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); - comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); - const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); - const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); - const valid = (half_inputs and opts.c_dtype == .f32) or - (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or - (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or - (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or - (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); - if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); - const c: u32 = switch (opts.c_dtype) { - .f16 => 0, - .f32 => 1, - .i32 => 2, - else => @compileError("unsupported tcgen05 accumulator dtype"), - }; - return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | - (@as(u32, @intFromBool(opts.a_negate)) << 13) | - (@as(u32, @intFromBool(opts.b_negate)) << 14) | - (@as(u32, @backingInt(opts.a_major)) << 15) | - (@as(u32, @backingInt(opts.b_major)) << 16) | - (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); - } - - fn inputFormat(comptime dtype: DType) u32 { - return switch (dtype) { - .f16, .f8e4m3fn => 0, - .bf16, .f8e5m2, .i8 => 1, - .f32 => 2, - .f4e2m1fn => 5, - else => @compileError("unsupported tcgen05 input dtype"), - }; - } -}; - -/// Typed description of a non-multicast tiled TMA atom: the element type, -/// bits moved per copy, basis of one box, and coordinate layout of the tensor. +/// Typed description of a tiled TMA atom: the element type, the bits moved +/// per copy, the basis of one box, and the coordinate layout of the tensor. pub const TmaConfig = struct { dtype: DType, copy_bits: u32, @@ -634,29 +475,18 @@ pub const TmaConfig = struct { coordinate_layout: LayoutSpec, coordinate_rank: u8, format: TmaFormat = .default, - /// Load completion group: with `.two`, each CTA's load completes on - /// the mbarrier of the pair's even CTA. Ignored by TMA stores. - cta_group: CtaGroup = .one, - mode: enum { tiled, gather4 } = .tiled, }; -pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; -pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; -pub const CopyBulkTensorTileS2GOp = struct {}; - pub const BlockScaledMma = struct { atom: Atom, tiled: Atom, }; -/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. -/// The instruction `kind` fixes B, the scales and K. +/// SM100 block-scaled MMA instruction: one CTA computes a 128 x `n` tile, +/// FP4 weights as A. The instruction `kind` fixes B, the scales and K. pub const BlockScaledMmaConfig = struct { kind: Kind = .mxfp4_mxfp8, - /// M is 128 for one CTA, or 128 or 256 for a CTA pair. - m: u16 = 128, n: u16, - cta_group: CtaGroup = .one, pub const Kind = enum { /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 @@ -694,23 +524,8 @@ pub const BlockScaledMmaConfig = struct { .nvfp4_nvfp4 => 16, }; } - - fn validate(self: BlockScaledMmaConfig) void { - const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); - const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; - if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { - std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); - } - } }; -fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { - return switch (config.cta_group) { - .one => .sm_90, - .two => .sm_100_2sm, - }; -} - fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { .u4 => .U4, @@ -758,7 +573,6 @@ pub const ArgSpec = struct { pub const MlirTypeSpec = struct { type_: *const mlir.Type, grid_constant: bool = false, - cuda_grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -867,7 +681,6 @@ pub const Builder = struct { .mlir_type => .{ .mlir_type = .{ .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, - .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, } }, } }; } @@ -894,7 +707,7 @@ pub const Builder = struct { return @Struct(.auto, null, &names, &types, &@splat(.{})); } - pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { std.debug.assert(self.entry_block == null); const ctx = self.ctx; const scratch = self.arena.allocator(); @@ -916,7 +729,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -1128,75 +941,6 @@ pub const Builder = struct { })); } - /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. - pub fn tensorMapType(self: *Builder) *const mlir.Type { - return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); - } - - /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / - /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. - fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { - const ctx = self.ctx; - const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); - const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); - const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ - .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, - .location = self.loc(), - })); - for (values, 0..) |v, i| { - const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ - .operands = .{ .flat = &.{array.inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, - .location = self.loc(), - })); - const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); - self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); - } - return array; - } - - /// Encode a tiled map in a host function and return both the map and CUDA status. - /// The caller supplies the final device address and handles the returned status. - pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { - const ctx = self.ctx; - // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. - ctx.setAllowUnregisteredDialects(true); - const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); - const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( - ctx, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, - self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, - addr.inner, - self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, - self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, - cuda.resultType(ctx), - self.tensorMapType(), - self.loc(), - )); - return .{ .map = results[1], .result = results[0] }; - } - - /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` - /// (`&tensor_map` for a `__grid_constant__` parameter). - pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { - const ctx = self.ctx; - const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); - return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ - .operands = .{ .flat = &.{llvm_ptr.inner} }, - .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, - .location = self.loc(), - })); - } - pub fn cudaResultStatus(self: *Builder, result: Value) Value { return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); } @@ -1275,8 +1019,7 @@ pub const Builder = struct { /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared /// memory, `i64` for global memory), for hand-written PTX operands. pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - const ptr = self.asValue(pointer); - return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); + return self.emit(cute_ir.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); } /// Reinterpret a pointer without changing its address. This is the typed @@ -1331,7 +1074,6 @@ pub const Builder = struct { const result = switch (e) { .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), - .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), .basis => |b| if (@hasDecl(A, "getBasis")) blk: { const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); @@ -1496,59 +1238,8 @@ pub const Builder = struct { // ==================== cute.core ==================== - fn layoutResult(comptime Shape: type) type { - return switch (@typeInfo(Shape)) { - .pointer => Layout, - else => View, - }; - } - - /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static - /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. - pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { - if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ - .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, - }); - var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - const shape_expr = self.coordExpr(&shape_values, shape); - const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { - var stride: AlgebraExpr = .{ .static = 1 }; - var stride_value: ?Value = null; - var index: usize = 0; - break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); - }; - const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); - const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); - const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); - const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { - if (shape == .tuple) { - const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); - for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); - return .{ .tuple = elements }; - } - if (shape == .static and shape.static == 1) return .{ .static = 0 }; - const result = stride.*; - if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); - if (shape == .static and stride.* == .static) { - stride.static *= shape.static; - } else { - const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { - const v: Value = .{ .inner = values[index.*], .kernel = self }; - index.* += 1; - break :blk v; - } else self.lift(shape.static); - stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); - stride.* = dynamicExpr(stride_value.*.?); - } - return result; - } - - fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + /// `cute.make_layout(shape, stride=...)`, static only. + pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); const stride = opts.stride orelse self.leftMost(shape); if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); @@ -1571,8 +1262,8 @@ pub const Builder = struct { fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { const T = @TypeOf(coord); if (T == Value) { - dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); - return dynamicExpr(coord); + dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); + return .{ .dynamic = 1 }; } if (T == AlgebraToken) return switch (coord) { .all => .all, @@ -1591,12 +1282,6 @@ pub const Builder = struct { } } - fn dynamicExpr(value: Value) AlgebraExpr { - std.debug.assert(value.isInt()); - const width = dtypeBitwidth(value.dtype()); - return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; - } - /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` /// placeholders. Integer leaves stay static in the dependent CuTe type. pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { @@ -1831,27 +1516,20 @@ pub const Builder = struct { /// `projection`, like Python's `proj`, selects the modes of `tile` that /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { - if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, - result_type.?, + result_type, .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - // A null result type on the algebra operations below requests dialect inference. - fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { - const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); - return .{ .inner = value.inner, .kernel = self }; - } - pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.local_partition( self.ctx, @@ -1864,13 +1542,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1889,17 +1566,13 @@ pub const Builder = struct { (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() else - std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); + std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| memref.ptr_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - desc.desc() else unreachable; const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); @@ -1907,12 +1580,11 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - result_type.?, + result_type, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1931,13 +1603,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1954,44 +1625,39 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - result_type.?, + result_type, .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -2028,94 +1694,44 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - result_type.?, + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), - result_type.?, + result_type, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - result_type.?, + result_type, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; - - pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { - var exec = atom; - if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .tiled, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable TMA store atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } - if (opts.tma_bar_ptr) |mbar| { - if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .gather4, - .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable gather atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .tiled, - .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable TMA atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } - exec = self.setTmaBarrier(exec, mbar); - } - if (opts.cache_policy) |policy| { - const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) - (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() - else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) - (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() - else - @panic("cache_policy requires an executable TMA copy atom"); - const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); - exec = .{ .inner = updated.inner, .kernel = self }; - } - self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { + self.emitVoid(cute_ir.copy(self.ctx, atom.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { @@ -2146,67 +1762,9 @@ pub const Builder = struct { // the kernel receives them as grid-constant arguments and makes them // executable before issuing copies. - pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; - - fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { - return self.emit(mlir.Operation.make(self.ctx, name, .{ - .operands = .{ .flat = operands }, - .attributes = attributes, - .result_type_inference = true, - .location = self.loc(), - })); - } - - fn valueShape(self: *Builder, value: Value) Value { - if (value.type_().isA(cute_ir.ShapeType) != null) return value; - return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); - } - - fn shapeRank(shape: Value) usize { - return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); - } - - /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout - /// at construction and passes data/index tensors together to partition/copy. - pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { - const Tiler = @TypeOf(cta_tiler); - const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) - self.asValue(cta_tiler) - else if (@typeInfo(Tiler) == .pointer) - self.makeShape(self.dimsExpr(cta_tiler)).value() - else - self.makeShape(cta_tiler).value(); - var smem = self.asValue(smem_layout); - const tiler_rank = shapeRank(self.valueShape(tiler)); - const smem_rank = shapeRank(self.valueShape(smem)); - std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); - if (smem_rank == tiler_rank + 1) { - const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); - for (modes, 0..) |*mode, i| mode.* = @intCast(i); - smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); - } - const shape = self.valueShape(self.asValue(gmem)); - const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); - const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); - var cfg = config; - cfg.mode = .tiled; - if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); - cfg.cta_group = op.cta_group; - if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); - if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); - cfg.mode = .gather4; - const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); - const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); - const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); - const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; - const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); - return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; - } - /// Build the host-visible TMA descriptor and its coordinate tensor, whose /// types are assembled from `config`. pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { - std.debug.assert(config.mode == .tiled); const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( self.ctx, self.asValue(gmem).inner, @@ -2214,7 +1772,7 @@ pub const Builder = struct { self.asValue(cta_map).inner, self.tmaLoadAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = .sm_90 }) catch @panic("invalid tiled TMA load")).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); @@ -2244,15 +1802,8 @@ pub const Builder = struct { /// Host-side tiled TMA load atom type described by `config`. pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ - .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .tmaGbasis = self.layoutType(config.global_basis), - .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), - }) catch @panic("invalid gather TMA atom")).type_(); return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ - .kind = tmaLoadKind(config), + .kind = .sm_90, .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), .tmaGbasis = self.layoutType(config.global_basis), @@ -2267,8 +1818,8 @@ pub const Builder = struct { return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), - .mode = if (config.mode == .gather4) .gather4 else .tiled, - .numCta = ctaCount(config.cta_group), + .mode = .tiled, + .numCta = 1, .gStride = self.emptyStrideType(), .tmaGbasis = self.layoutType(config.global_basis), }) catch @panic("invalid TMA load atom")).type_(); @@ -2300,12 +1851,6 @@ pub const Builder = struct { return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); } - /// Generic address (i64) of the tensor map in an executable tiled TMA atom. - pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { - const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); - return self.ptrToInt(ptr, .i64); - } - pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); } @@ -2316,11 +1861,11 @@ pub const Builder = struct { } /// Attach the mbarrier that an executable TMA load atom completes. - pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { + pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, atom.inner, - mbar.inner, + barrier.inner, atom.type_(), (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), self.loc(), @@ -2370,10 +1915,10 @@ pub const Builder = struct { } fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - cfg.validate(); + if (cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) std.debug.panic("unsupported block-scaled MMA N {d}", .{cfg.n}); return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), - .numCta = ctaCount(cfg.cta_group), + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ 128, cfg.n, cfg.k() })).attribute(), + .numCta = 1, .aMajor = .k, .bMajor = .k, .aType = DType.f4e2m1fn.toMlir(self.ctx), @@ -2407,27 +1952,10 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub const Tcgen05CommitOptions = struct { - /// Must match the group of the operations being tracked. - cta_group: CtaGroup = .one, - /// Arrive at the same barrier offset in each CTA whose rank bit is set. - /// Without a mask, arrive only on the issuing CTA's barrier. - multicast_mask: ?Value = null, - }; - - /// Track this thread's preceding tcgen05 MMA/copy operations of the - /// selected group, signaling one arrival per destination on completion. - /// The same elected thread that issues those operations must commit them. - /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 - /// operations in a kernel must use the same group. - pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { - self.emitVoid(nvvm.tcgen05_commit( - self.ctx, - self.sharedBarrierPtr(mbar).inner, - opts.cta_group.toNvvm(), - if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, - self.loc(), - )); + /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected + /// lane of the MMA warp issues this after each K-stage. + pub fn tcgen05Commit(self: *Builder, barrier: Value) void { + self.emitVoid(nvvm.tcgen05_commit(self.ctx, self.sharedBarrierPtr(barrier).inner, .cta_1, self.loc())); } /// Wait until asynchronous tensor-memory loads have reached registers. @@ -2435,14 +1963,14 @@ pub const Builder = struct { self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); } - pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; + pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, self.constant(.i32, columns).inner, self.asValue(holding_ptr).inner, - .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2453,7 +1981,7 @@ pub const Builder = struct { } pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null }, self.loc())); } pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { @@ -2461,7 +1989,7 @@ pub const Builder = struct { self.ctx, self.asValue(pointer).inner, self.constant(.i32, columns).inner, - .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2654,37 +2182,22 @@ pub const Builder = struct { _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); } - /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). - /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be - /// a positive warp multiple, and every participating thread must execute - /// the same barrier id. The aligned form requires CTA-uniform control flow. - pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { - if (@TypeOf(participants) == ?Value) { - if (participants) |count| return self.barrierCtaSyncAligned(id, count); - return self.barrierCtaSyncAligned(id, null); - } - if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { - std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); - } - if (@TypeOf(id) == Value) { - std.debug.assert(id.dtype() == .i32); - } else { - const immediate: i32 = id; - std.debug.assert(immediate >= 0 and immediate < 16); - } - // The LLVM intrinsic preserves convergence and both operands across - // the embedded compiler's NVVM dialect version boundary. - if (@TypeOf(participants) == @TypeOf(null)) { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); - } else { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); - } - } - - pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; - - pub fn barrier(self: *Builder, opts: BarrierOptions) void { - self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); + /// Synchronize a fixed subset of the CTA on a named hardware barrier. + /// `participants` must be a warp multiple and every participating thread + /// must execute the same barrier id. + pub fn namedBarrier(self: *Builder, id: i32, participants: i32) void { + // The embedded CuTe compiler currently reads NVVM through a different + // MLIR revision. A generic `nvvm.barrier` with optional operands is + // accepted, but those operands are discarded and it lowers to + // `bar.sync 0`. Constant inline PTX preserves the named barrier ABI + // used by CUTLASS (`2,160` for TMEM allocation and `1,128` for the + // epilogue pipeline). + const assembly = std.fmt.allocPrint( + self.arena.allocator(), + "bar.sync {d}, {d};", + .{ id, participants }, + ) catch @panic("namedBarrier OOM"); + _ = llvm.inline_asm(self.ctx, assembly, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2855,101 +2368,48 @@ pub const Builder = struct { /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe /// shared-memory pointer or a raw `i32` shared-memory address. - fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { - if (mbar.isInt()) return self.addressPtr(mbar, .shared); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { + if (barrier.isInt()) return self.addressPtr(barrier, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{barrier.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. - pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); + pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); } - pub const MBarrierArriveOptions = struct { - peer_cta_rank_in_cluster: ?Value = null, - arrive_count: ?Value = null, - scope: MBarrierScope = .cta, - relaxed: bool = false, - }; - - /// Arrive locally or at the same offset in a peer CTA. Local arrivals - /// return the opaque state token; remote arrivals have no state result. - pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { - var ptr = self.sharedBarrierPtr(mbar); - if (opts.peer_cta_rank_in_cluster) |rank| { - ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); - } - const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ - .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, - .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { - .cta => .cta, - .cluster => .cluster, - } }) catch unreachable).attribute(), - .relaxed = .boolean(self.ctx, opts.relaxed), - .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, - }, self.loc()); - if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); - self.emitVoid(op); - return null; + pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { + self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, count).inner, .cta, self.loc())); } - pub const MBarrierTryWaitOptions = struct { - scope: MBarrierScope = .cta, - time_limit: ?Value = null, - }; - - /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend - /// briefly and return false before completion. `scope` selects which - /// threads this wait can synchronize with; the barrier remains local. - pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { - const ptr = self.sharedBarrierPtr(mbar); - if (opts.time_limit) |ticks| { - return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); - } - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); + /// Test whether phase `phase` has completed, without suspending (SM90+). + pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, .@"try", .cta, self.loc())); } - pub const MBarrierWaitOptions = struct { - scope: MBarrierScope = .cta, - /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. - time_limit: u32 = 10_000_000, - /// Probe before suspending, as in CuTe's pipeline consumer wait. - test_first: bool = true, - }; - - fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { - const ptr = self.sharedBarrierPtr(mbar); - const ticks = self.constant(.i32, opts.time_limit); - switch (opts.scope) { - // Preserve the operation recognized by CuTe's pipeline compiler. - .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), - .cluster => { - var pending = self.openWhile(.{}, .{}); - const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); - pending.yieldBefore(ready.eq(false), .{}); - pending.yieldAfter(.{}); - }, - } + /// Wait for phase `phase`, letting the warp sleep for up to + /// `suspend_time` cycles per attempt (SM90+). + pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { + self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.constant(.i32, suspend_time).inner, self.loc())); } - /// Acquire a pipeline phase at `scope`: test without suspending, then - /// block only when the phase is still outstanding. The initial test matters when + /// Wait for a pipeline phase with the two-step sequence of CuTe's + /// `PipelineTmaUmma`: test the barrier without suspending, then suspend + /// only when the phase is still outstanding. The initial test matters when /// a producer reaches a full ring and must yield to the consumer that /// recycles its next stage. - pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { - if (opts.test_first) { - const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); - var pending = self.openIf(ready.eq(false)); - self.mbarrierWaitLoop(mbar, phase, opts); + pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { + const ready = self.mbarrierWaitParity(barrier, phase); + var pending = self.openIf(ready.eq(false)); + { + self.mbarrierTryWaitParity(barrier, phase, 10_000_000); pending.close(); - } else { - self.mbarrierWaitLoop(mbar, phase, opts); } } @@ -2978,7 +2438,7 @@ pub const Builder = struct { // Hand-scheduled kernels address shared memory with `i32` offsets in the // shared window, global memory with `i64` addresses and tensor memory with // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / - // LLVM operations for them directly, using inline PTX where needed. + // LLVM operations for them directly (no inline PTX). pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; @@ -3040,6 +2500,11 @@ pub const Builder = struct { self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); } + /// `bar.sync id, threads`: named barrier over `threads` threads. + pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads) }, null); + } + /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether /// `predicate` (i1) is true in any of them. pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { @@ -3051,10 +2516,9 @@ pub const Builder = struct { self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); } - /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). - pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { - std.debug.assert(opts.time_limit == null); - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); + /// Non-blocking test of mbarrier phase `parity` (i1). + pub fn mbarrierTestParity(self: *Builder, barrier: Value, parity: Value) Value { + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, parity.inner, .@"test", .cta, self.loc())); } /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of @@ -3085,8 +2549,8 @@ pub const Builder = struct { /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with /// `noinc`, the arrival is counted in the barrier's expected count. - pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { - _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); + pub fn cpAsyncMbarrierArrive(self: *Builder, barrier: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(barrier)}, null); } /// Wait until all of this thread's cp.async copies have landed. @@ -3241,271 +2705,13 @@ pub const Builder = struct { return self.gridDim().z.mul(value); } - /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). - /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), - /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). - /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` - /// are element coordinates; row length and out-of-bounds fill come from the - /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. - pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { - std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); - for (r) |row| std.debug.assert(row.dtype() == .i32); - _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; - - /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. - pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { - if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); - const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); - self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ - .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, - }, self.loc())); - } - - pub fn prefetchL1(self: *Builder, addr: Value) void { - const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); - _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn prefetchTensorMap(self: *Builder, desc: Value) void { - const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); - _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub const OperandSource = union(enum) { smem: Value, tmem: Value }; - pub const Tcgen05MmaWsOptions = struct { - collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, - collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, - col_b_zero_mask: ?Value = null, - }; - - /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. - /// The caller elects the issuer and commits/waits before consuming D. - pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { - std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); - const adesc = switch (a) { - .smem => |desc| self.smemDescriptorToInt(desc), - .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), - }; - const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); - self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ - .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, - .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, - .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, - }, self.loc())); - } - - pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { - return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { - .k => .k, - .mn => .mn, - } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); - } - - pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); - } - - pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { - if (desc.isInt()) return desc.to(.i64); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); - } - - pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { - self.setMaxRegister(count, .increase); - } - - pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { - self.setMaxRegister(count, .decrease); - } - - fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { - comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); - const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); - _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. - pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { - std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); - _ = llvm.inline_asm(self.ctx, switch (scope) { - .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", - .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", - }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. - fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { - if (!ptr.isPtr()) return ptr; - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); - } - - /// Map a local shared pointer/address to a typed pointer in peer CTA memory. - pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { - const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); - const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); - if (smem.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem); - const result_type = (cute_ir.PtrType.get(self.ctx, .{ - .valueType = ty.valueType(), - .addressSpace = .dsmem, - .alignment = ty.alignment(), - .swizzle = ty.swizzle(), - .bitlayout = ty.bitlayout(), - }) catch @panic("invalid DSMEM pointer")).type_(); - return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); - } - return remote; - } - - /// Store one, two, or four i32 words in a peer CTA, completing bytes on its - /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. - pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { - const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); - comptime std.debug.assert(n == 1 or n == 2 or n == 4); - if (smem.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); - } - if (mbar.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); - } - const packed_value = if (@TypeOf(value) == Value) value else blk: { - var words: [n]Value = undefined; - inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); - break :blk self.joinWords(n, words); - }; - self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); - } - - /// Explicit aligned variants retain the convergence requirement of raw PTX. - pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn barrierClusterWaitAligned(self: *Builder) void { - self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); - } - - pub const ClusterArriveOptions = struct { aligned: bool = false }; - - pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { - if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); - _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn clusterWait(self: *Builder) void { - self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); - } - - /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. - pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { - const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); - return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; - } - - /// A null mask selects CuTe's FULL_MASK. - pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { - const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); - return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; - } - - pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; - - pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { - if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; - return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ - .abs = if (opts.abs) .unit(self.ctx) else null, - .nan = if (opts.nan) .unit(self.ctx) else null, - .ftz = if (opts.ftz) .unit(self.ctx) else null, - }, self.loc())); - } - - fn vec2(self: *Builder, x: [2]Value) Value { - const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); - return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); - } - - fn unvec2(self: *Builder, v: Value) [2]Value { - return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; - } - - pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; - - fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { - return .{ - .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), - .ftz = if (opts.ftz) .unit(self.ctx) else null, - }; - } - - pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub const StoreOptions = struct { - /// Address space for raw integer addresses; typed pointers retain theirs. - space: NVPTXAddressSpace = .global, - level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, - }; - - /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. - pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { - if (@TypeOf(value) != Value) { - const n = common.arity(@TypeOf(value), "store: value"); - comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); - var words: [n]Value = undefined; - inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); - const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; - if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { - const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); - var args: [9]*const mlir.Value = undefined; - args[0] = address.inner; - for (words, 0..) |word, i| args[i + 1] = word.inner; - _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - return; - } - self.store(addr, self.joinWords(n, words), opts); - return; - } - const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { - const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); - break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); - } else addr; - self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ - .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), - }, self.loc())); - } - - pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { - return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; - } - - pub fn clock(self: *Builder) Value { - return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); - } - - pub fn globalTimer(self: *Builder) Value { - return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); - } - // ==================== scalars ==================== - /// `value` as a constant of `dt`; a Value passes through and enums use - /// their backing integers. An integer is signed or an unsigned bit - /// pattern of `dt`'s width; anything else panics. + /// `value` as a constant of `dt`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dt`'s width; anything else + /// panics. pub fn constant(self: *Builder, dt: DType, value: anytype) Value { if (@TypeOf(value) == Value) return value; - if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); const ctx = self.ctx; const T = @TypeOf(value); if (isFloatDtype(dt)) { @@ -3751,7 +2957,7 @@ test "guard, shared memory, sync, layouts, for loop" { const i = b.blockIdx().x.mul(128).add(tid); smem.set(.{tid}, gx.get(.{i})); b.syncThreads(); - b.barrierCtaSyncAligned(2, 128); + b.namedBarrier(2, 128); b.launchDependents(); b.waitForDependency(); const j = b.constant(.i32, 127).sub(tid); @@ -3770,9 +2976,7 @@ test "guard, shared memory, sync, layouts, for loop" { try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); - try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); - try expectContains(kernel, "value = 2 : i32"); - try expectContains(kernel, "value = 128 : i32"); + try expectContains(kernel, "bar.sync 2, 128;"); try expectContains(kernel, "griddepcontrol.launch_dependents;"); try expectContains(kernel, "griddepcontrol.wait;"); try expectContains(kernel, "scf.if"); @@ -3909,1045 +3113,30 @@ test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); defer b.deinit(); - const tma: TmaConfig = .{ - .dtype = .f8e4m3fn, - .copy_bits = 16384, - .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), - .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), - .coordinate_rank = 3, - }; - var pair_tma = tma; - pair_tma.cta_group = .two; - const args = try b.declareArgs(.{ - .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, - .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, + const descriptor = try cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(ctx, .{ + .kind = .sm_90, + .valType = DType.f8e4m3fn.toMlir(ctx), + .copyBits = 16384, + .tmaGbasis = b.layoutType(b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) })), + .tmaFormat = .U8, }); - _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); - _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); + _ = try b.declareArgs(.{ .descriptor = .{ .mlir_type = .{ .type_ = descriptor.type_(), .grid_constant = true } } }); const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - for ([_]CtaGroup{ .one, .two }) |group| { - _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); - } - _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); const ir = try b.finish(.{ 192, 1, 1 }); defer std.testing.allocator.free(ir); - try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); - try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); - try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); - try expectContains(ir, "non_exec_tiled_tma_load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); const parsed = try mlir.Module.parse(ctx, ir); defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); } - -test "barrier helpers preserve dynamic ids, CTA groups, masks and remote scopes" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "barrier_operands"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - - const barrier = b.allocSmem(.i64, 1, 8); - const peer = b.clusterCtaRank().bitXor(1); - b.barrierCtaSyncAligned(2, 128); - b.barrierCtaSyncAligned(b.threadIdx().x.div(128).mul(2).add(1), 128); - b.tcgen05Commit(barrier, .{ .cta_group = .two }); - b.tcgen05Commit(barrier, .{ .cta_group = .two, .multicast_mask = b.constant(.i32, 3) }); - _ = b.mbarrierArrive(barrier, .{ .peer_cta_rank_in_cluster = peer, .arrive_count = b.constant(.i32, 1), .scope = .cta }); - _ = b.mbarrierArrive(barrier, .{ .peer_cta_rank_in_cluster = peer, .arrive_count = b.constant(.i32, 32), .scope = .cluster }); - - const ir = try b.finish(.{ 256, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "llvm.nvvm.barrier.cta.sync.aligned.count")); - try expectContains(ir, "value = 2 : i32"); - try expectContains(ir, "value = 128 : i32"); - try expectContains(ir, "#nvvm.cta_group"); - try expectContains(ir, "value = 3 : i32"); - try expectContains(ir, "value = 32 : i32"); - try expectContains(ir, "!llvm.ptr<7>"); - try expectContains(ir, "#nvvm.mem_scope"); - try expectContains(ir, "#nvvm.mem_scope"); - - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "mbarrier waits preserve acquire scope through polling and blocking retries" { - const ctx = try testContext(); - defer ctx.deinit(); - for ([_]MBarrierScope{ .cta, .cluster }) |scope| { - var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - - const barrier = b.allocSmem(.i64, 1, 8); - const phase = b.constant(.i32, 0); - _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); - _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); - b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); - b.mbarrierWait(barrier, phase, .{ .scope = scope }); - - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); - try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); - try expectContains(ir, "value = 1234 : i32"); - try expectContains(ir, "\"scf.if\""); - switch (scope) { - .cta => { - try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); - }, - .cluster => { - try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - }, - } - - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); - } -} - -test "SM100 raw primitives preserve operand types and round-trip through MLIR" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f8e4m3fn, - .copy_bits = 1024, - .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - }; - const a = try b.declareArgs(.{ - .gmem = .{ .ptr = DType.i32 }, - .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, - }); - const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); - const desc = b.tmaDescriptorAddress(atom); - const dst = b.constant(.i32, 1024); - const bar = b.constant(.i32, 0); - const zero = b.constant(.i32, 0); - const gmem = b.ptrToInt(a.gmem, .i64); - const pred = b.threadIdx().x.eq(0); - b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); - b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); - b.prefetchL1(gmem); - b.prefetchTensorMap(desc); - const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); - const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); - b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); - b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); - b.setMaxRegisterIncrease(128); - b.barrierCtaSyncAligned(0, 512); - b.setMaxRegisterDecrease(128); - b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); - b.mbarrierSpin(bar, zero, .cta); - b.mbarrierSpin(bar, zero, .cluster); - const state = b.mbarrierArrive(bar, .{}).?; - const remote = b.mapDsmemPtr(dst, zero); - _ = remote; - _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); - b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); - b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); - b.barrierClusterArriveRelaxedAligned(); - b.barrierClusterWaitAligned(); - const any = b.voteAnySync(pred, null); - const mask = b.voteBallotSync(pred, null); - const x = b.constant(.f32, 1.0); - const y = b.constant(.f32, 2.0); - const maximum = b.fmax(x, y, .{}); - const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); - const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); - const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); - const scalar = b.fma(maximum, fused[0], fused[1]); - b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); - - const ir = try b.finish(.{ 512, 1, 1 }); - defer std.testing.allocator.free(ir); - for ([_][]const u8{ - "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", - "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", - "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", - "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", - "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", - "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", - "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", - "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", - "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", - "%globaltimer", - }) |op| try expectContains(ir, op); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { - try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); - try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); - try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); - try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); -} - -test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); - defer b.deinit(); - const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); - const addr = b.constant(.i32, 128); - const fields = [_]u64{ - 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes - 0x4002404000010000, // sw128, pattern at byte 128: base 1 - 0x400e404000010000, // sw128, pattern at byte 896: base 7 - 0x00007fff3fff0000, // unswizzled, largest representable offsets - }; - const descs = [_]Value{ - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), - }; - const out = b.ptrToInt(args.out, .i64); - for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - for (fields) |bits| { - const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); - defer std.testing.allocator.free(expected); - try expectContains(ir, expected); - } - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const n = b.blockIdx().x.add(1); - const singleton = b.constant(.i32, 1); - const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); - const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); - const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); - _ = compact; - _ = explicit; - _ = static_nested; - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "((4,?),?):((1,4),?)"); - try expectContains(ir, "(?,16):(1,?)"); - try expectContains(ir, "((4,8),16):((1,4),32)"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "makeLayout preserves 64-bit shape and stride values" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); - defer b.deinit(); - const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); - const wide = args.extent.add(@as(i64, 1) << 32); - const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); - const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; - try std.testing.expect(explicit.type_().eql(b.layoutType(.{ - .shape = b.expr(.{ 8, 2 }), - .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, - }))); - _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); - try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); - try expectContains(ir, "value = 4294967296 : i64"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f32, - .copy_bits = 2048, - .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - }; - const args = try b.declareArgs(.{ - .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, - }); - const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); - const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); - const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); - const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); - b.copy(store.atom, .{smem}, .{tile}, .{}); - b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); - const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); - _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); - _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); - try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); - try expectContains(ir, "(16,4):(1@0,1@1)"); - try expectContains(ir, "tma_store"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); - defer b.deinit(); - const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); - const tid = b.threadIdx().x; - const mbar = b.allocSmem(.i64, 1, 8); - b.barrier(.{}); - b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); - b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); - _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); - const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; - b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); - b.clusterArriveRelaxed(.{}); - b.clusterWait(); - b.clusterArriveRelaxed(.{ .aligned = true }); - b.barrierClusterWaitAligned(); - _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); - _ = b.voteBallotSync(tid.eq(0), 0xff); - const x = tid.to(.f32); - const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); - const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); - b.store(a.out, state.to(.i32), .{}); - b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); - const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); - const raw_desc = b.smemDescriptorToInt(desc); - _ = b.intToSmemDescriptor(raw_desc); - const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); - const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; - try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); - try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); - b.store(mapped, tid, .{}); - const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; - for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); - const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); - b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); - b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); - const ir = try b.finish(.{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); - try expectContains(ir, "nvvm.store.ext"); - try expectContains(ir, "vector<2xi32>"); - try expectContains(ir, "#nvvm.fp_rnd_mode"); - try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f32, - .copy_bits = 2048, - .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - .mode = .gather4, - }; - var pair = cfg; - pair.cta_group = .two; - const args = try b.declareArgs(.{ - .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, - .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, - .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, - }); - const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); - const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); - const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); - const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; - const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); - const mbar = b.allocSmem(.i64, 1, 8); - b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ - .tma_bar_ptr = mbar, - .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), - }); - b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); - b.endFunction(.{ 32, 1, 1 }); - - b.beginFunction("gather4_copy", .host); - const host = try b.declareArgs(.{ - .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, - .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, - }); - const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); - const tiler = b.makeTile(&.{ 16, 4 }); - _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); - _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); - b.returnHostStatus(b.constant(.i32, 0)); - b.endFunction(null); - const ir = try b.finishProgram(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); - try expectContains(ir, "atom.tma_partition"); - try expectContains(ir, "mode = gather4, num_cta = 1"); - try expectContains(ir, "mode = gather4, num_cta = 2"); - try expectContains(ir, "non_exec_2d_gather4_tma_load", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); - const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); - const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); - for (out, 0..) |*o, i| { - o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{packed_.inner} }, - .results = .{ .flat = &.{elem} }, - .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, - .location = b.loc(), - })); - } - return out; -} - -pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { - var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); - for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); - return w.written(); -} - -pub fn ldAcq(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); -} - -pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); -} - -pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { - return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); -} - -pub fn fenceAcqRelCta(b: *Builder) void { - asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); -} - -pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { - if (@TypeOf(value) == Value) return value; - return b.constant(dt, value); -} - -/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand -/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. -pub fn zext64(b: *Builder, v: Value) Value { - if (v.dtype() == .i64) return v; - return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); -} - -pub fn i16Of(b: *Builder, v: Value) Value { - return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); -} - -pub fn cst32(b: *Builder, v: anytype) Value { - return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); -} - -pub fn cst64(b: *Builder, v: u64) Value { - return b.constant(.i64, @as(i64, @bitCast(v))); -} - -pub fn getSmIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); -} - -pub fn getLaneIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); -} - -/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. -pub fn getGridIdx(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); -} - -pub fn blockRankInCluster(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); -} - -/// `clock64()` (`mov.u64 %rd, %clock64;`). -pub fn clock64(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); -} - -/// Keep the original election sequence until typed election matches its small-batch performance. -pub fn electOneSync(b: *Builder) Value { - const pred = asmValue(b, - \\{ - \\.reg .b32 %rx; - \\.reg .pred %px; - \\ mov.u32 $0, 0; - \\ elect.sync %rx|%px, $1; - \\@%px mov.s32 $0, 1; - \\} - , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); - return pred.ne(0); -} - -/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). -pub fn trap(b: *Builder) void { - asmVoid(b, "trap;", &.{}, "~{memory}"); -} - -/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may -/// reach it at different program points). -pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { - asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); -} - -/// `cute::cluster_arrive`. -pub fn clusterArrive(b: *Builder) void { - asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); -} - -/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). -pub fn gridDependencySynchronize(b: *Builder) void { - asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::warpgroup_reg_alloc`. -pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `cutlass::arch::warpgroup_reg_dealloc`. -pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `ptx::ld_volatile(const uint64_t*)`. -pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); -} - -/// `ptx::ld_volatile(const uint32_t*)`. -pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const uint32_t*)`. -pub fn ldSharedU32(b: *Builder, smem: Value) Value { - return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const float2*)`. -pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { - return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); -} - -/// `ptx::ld_shared(const float4*)`. -pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); -} - -/// `ptx::ld_shared(const uint4*)`. -pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); -} - -/// `ptx::st_shared(const void*, x, y, z, w)`. -pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { - asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); -} - -/// `ptx::st_shared(const uint32_t*, uint32_t)`. -pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { - asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); -} - -/// `ptx::st_shared(const float2*, float2)`. -pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { - asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); -} - -/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. -pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { - std.debug.assert(num_bytes % 8 == 0); - asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); -} - -/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. -pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { - return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); -} - -/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte -/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its -/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). -pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { - std.debug.assert(words.len % 4 == 0); - const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); - const mapped_dst = mapaShared(b, dst, dst_cta_idx); - var i: usize = 0; - while (i < words.len) : (i += 4) { - asmVoid( - b, - "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", - &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, - "r,r,r,r,r,r,~{memory}", - ); - } -} - -/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. -pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); -} - -/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. -pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); -} - -/// `ClusterBarrier::arrive()` on this CTA's barrier. -pub fn mbarrierArrive(b: *Builder, bar: Value) void { - asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); -} - -/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of -/// cluster CTA `cta_id`. -pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { - asmVoid( - b, - "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", - &.{ bar, cta_id }, - "r,r,~{memory}", - ); -} - -/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA -/// `cta_id`'s barrier, only when `pred == 1`. -pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { - asmVoid( - b, - "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", - &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, - "r,r,r,r,~{memory}", - ); -} - -/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). -pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { - asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); -} - -/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of -/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. -pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, - "r,l,r,r,r,l,~{memory}", - ); -} - -/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. -pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, - "r,l,r,r,r,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of -/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. -pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, - "r,l,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. -pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, - "l,r,r,l,~{memory}", - ); -} - -/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. -pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { - asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. -pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { - asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. -pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { - asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `ptx::tcgen05_before_thread_sync`. -pub fn tcgen05BeforeThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); -} - -/// `ptx::tcgen05_after_thread_sync`. -pub fn tcgen05AfterThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). -pub fn fenceViewAsyncTmemLoad(b: *Builder) void { - asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); -} - -/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so -/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). -pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { - comptime var constraints: []const u8 = ""; - inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; - inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); - constraints = constraints ++ ",~{memory}"; - return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); -} - -/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks -/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value -/// tied through it. -pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { - const n = values.len; - const a = b.arena.allocator(); - var w: std.Io.Writer.Allocating = .init(a); - for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); - for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); - w.writer.writeAll(",~{memory}") catch @panic("OOM"); - return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); -} - -/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets -/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. -pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { - return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); -} - -/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. -pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { - asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); -} - -/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, -/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) -/// nonzero accumulates. -pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { - const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); - asmVoid( - b, - "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", - &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, - "r,l,l,r,r,r,r", - ); -} - -/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, -/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. -pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { - var elected = b.openIf(electOneSync(b)); - { - asmVoid( - b, - "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", - &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, - "r,h,~{memory}", - ); - elected.close(); - } -} - -pub fn bits32(v: Value) Value { - return if (v.dtype() == .i32) v else v.bitCast(.i32); -} - -pub fn fromBits32(v: Value, dt: DType) Value { - return if (dt == .i32) v else v.bitCast(dt); -} - -/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). -pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { - const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. -pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { - const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_up_sync(0xffffffff, v, delta)`. -pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { - const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__reduce_add_sync(0xffffffff, v)` (u32). -pub fn reduceAddSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__reduce_min_sync(0xffffffff, v)` (u32). -pub fn reduceMinSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). -pub fn allSync(b: *Builder, pred: Value) Value { - const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); - return r.ne(0); -} - -pub fn popc(b: *Builder, v: Value) Value { - return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; -} - -/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). -pub fn ffs(b: *Builder, v: Value) Value { - return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); -} - -/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. -pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { - return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); -} - -/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). -pub fn fminf(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -/// `fabsf(x)` (`abs.f32`). -pub fn fabsf(b: *Builder, x: Value) Value { - return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); -} - -/// `math::fast_rcp(x)`. -pub fn fastRcp(b: *Builder, x: Value) Value { - return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); -} - -/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). -pub fn expfFast(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). -pub fn expfFastNeg(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. -pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { - return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); -} - -/// `__hmin2(a, b)` on bf16x2 words. -pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__hmax2(a, b)` on bf16x2 words. -pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). -pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { - return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); -} - -/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with -/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). -pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { - const lo = i16Of(b, bf16x2); - const hi = i16Of(b, bf16x2.shrLogical(16)); - return .{ - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), - }; -} -/// IEEE operations used when staging quantized inputs; no FTZ or contraction. -pub fn divRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} -pub fn mulRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -test "canonical primitives preserve aligned barriers and runtime transaction counts" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const bar = b.constant(.i32, 0); - const phase = b.constant(.i32, 0); - const bytes = b.threadIdx().x.add(16); - b.barrierCtaSyncAligned(1, 32); - b.barrierClusterArriveRelaxedAligned(); - b.barrierClusterWaitAligned(); - b.mbarrierInit(bar, 1); - b.fenceMbarrierInit(); - b.mbarrierArriveExpectTx(bar, bytes); - b.mbarrierArriveExpectTx(bar, 16); - b.mbarrierWait(bar, phase, .{ .test_first = false }); - _ = phase.bitXor(1); - b.fenceProxyAsyncShared(); - b.fenceProxyAsyncShared(); - b.cpAsyncBulkCommitGroup(); - b.cpAsyncBulkWaitGroup(0, false); - _ = b.voteBallotSync(bytes.ne(0), null); - _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); - try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); - try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} diff --git a/dsl/fly/README.md b/dsl/fly/README.md index 687c99dc..b886988b 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel/fly/test.zig`. +and `zml/kernel.zig`. Model kernels live beside their consumers: diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index b01b7189..ec1ae61f 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,7 +104,8 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. + the default `.broadcast` keeps a copy per chip. Call it inside + `zml.ops.manualComputation` so it sees each partition's shards. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 7ff70707..b78e230b 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -65,14 +65,14 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module const run_start: std.Io.Timestamp = .now(io, .awake); - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index d471f74d..adc3882b 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -9,3 +9,10 @@ zig_binary( deps = ["//zml"], visibility = ["//visibility:public"], ) + +# Tooling: the loader's validation harness. Its header lists its knobs. +zig_binary( + name = "load_check", + main = "load_check.zig", + deps = ["//zml"], +) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig new file mode 100644 index 00000000..ad98ce6a --- /dev/null +++ b/examples/io/load_check.zig @@ -0,0 +1,349 @@ +//! Loads every tensor of a checkpoint with `zml.io.Loader` and checks the +//! bytes the devices hold against the checkpoint's. Tooling, not an +//! example: the harness the loader's backends are validated with on each +//! host. +//! +//! bazel run //examples/io:load_check -- [replicated|sharded|tp] +//! bazel run //examples/io:load_check -- dma-bench +//! +//! The checkpoint is a model directory, a .safetensors file or a +//! .safetensors.index.json, local or remote (hf://, s3://, gs://, https://, +//! with the credentials and proxies of the environment). Each tensor is +//! placed on the `tp` mesh, all the platform's devices on its `model` axis, +//! by the mode: +//! - `replicated`: whole on every device; +//! - `sharded`, the default: split along its first axis, which the devices +//! must divide, whatever the model (DeepSeek-V4.x cannot bind on 2 or 4 +//! devices); +//! - `tp`: the input axis of `o_proj` and `down_proj` weights, as tensor +//! parallelism shards them (column-sharded: every device's bytes are +//! strided in the checkpoint), the first axis of the others. +//! A scalar is replicated in every mode. +//! +//! `dma-bench` loads nothing: it runs the direct backend's DMA calibration +//! (`zml.io.dma_calibration.measure`) on the platform and logs the +//! block it selects. A platform that loads through the buffered backend has +//! no DMA to measure, and CPU selects the smallest block unmeasured. Its +//! knobs, beside the platform's below, are the screen's options, at their +//! defaults when unset: +//! - `ZML_DMA_BENCH_BLOCK_MIB=a,b,...`: the candidate blocks in MiB, each +//! at most 32 (2, 4, 8, 16 and 32); a load takes a single one (below); +//! - `ZML_DMA_BENCH_BLOCK_PARALLELISM=n`: transfers in flight per device, +//! at most 32 (8); +//! - `ZML_DMA_BENCH_WINDOW_MS=n`: a screen window's minimum duration (2); +//! - `ZML_DMA_BENCH_MIN_TRANSFERS=n`: the transfers a window waits for on +//! the measured device, 0 for none (32); +//! - `ZML_DMA_BENCH_BLOCK_TOLERANCE=f`: the smallest block within this +//! fraction of the peak rate is selected (0.08). +//! +//! The loader logs the bytes it loaded, its time and rate at info, and its +//! stats at debug, on the `zml/io/load` scope; `zml/io` logs the DMA +//! calibration's pick with its measured rate on one device (`dma_bench`); +//! `zml/vfs` logs the remote read totals. This tool enables all three. +//! +//! Knobs, from the environment: +//! - `ZML_LOAD_CHECK=n`: once loaded, read every n-th tensor and the largest +//! one back to host and compare them with the checkpoint's bytes, every +//! replica of a tensor each device holds whole. 1 checks every tensor; +//! unset or 0, none. A difference is `LoadContentMismatch`, logged. +//! - `ZML_LOAD_READ_PARALLELISM=n`: the loader's concurrent source reads +//! (`Loader.Options.read_parallelism`); unset, the checkpoint's default +//! width, 16 locally and 32 remotely. +//! - `ZML_DMA_BENCH_BLOCK_MIB=n`: the direct backend's DMA block, in MiB +//! (`Loader.Options.dma_block_size`); unset, calibration picks one. +//! - `ZML_DIRECT_IO=off|on|auto`: direct I/O for local checkpoint files +//! (`Loader.Options.direct_io`, `VFS.DirectIo`); `auto` when unset. +//! - `ZML_LOAD_MEMORY=default|host_pinned|host_mapped`: where every tensor +//! is delivered (`TensorStore.Memory`, `View.setMemory`); `default`, device +//! memory, when unset. The loader refuses a kind the platform cannot take +//! (`InvalidBinding`). +//! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction +//! of each device's memory on CUDA, ROCm and oneAPI, instead of the +//! platform's allocator defaults, for a run that observes the loader's +//! admission on a device with room. +//! - `ZML_CPU_DEVICE_COUNT=n`: the CPU platform's devices (4 by default). +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.load_check); + +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/io/load", .level = .debug }, + .{ .scope = .@"zml/io", .level = .debug }, + .{ .scope = .@"zml/vfs", .level = .debug }, + }, +}; + +const Mode = enum { replicated, sharded, tp }; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var args = init.minimal.args.iterate(); + _ = args.next(); // skip program name + const path = args.next() orelse { + log.err("usage: load_check [replicated|sharded|tp], or load_check dma-bench", .{}); + return error.MissingPath; + }; + if (std.mem.eql(u8, path, "dma-bench")) return dmaBench(init); + const mode = std.meta.stringToEnum(Mode, args.next() orelse "sharded") orelse { + log.err("the mode must be replicated, sharded or tp", .{}); + return error.InvalidArgument; + }; + const check_stride = try envNumber(usize, init.environ_map, "ZML_LOAD_CHECK") orelse 0; + const direct_io = std.meta.stringToEnum(zml.io.VFS.DirectIo, init.environ_map.get("ZML_DIRECT_IO") orelse "auto") orelse { + log.err("ZML_DIRECT_IO must be off, on or auto", .{}); + return error.InvalidArgument; + }; + const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); + const memory = std.meta.stringToEnum(zml.io.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { + log.err("ZML_LOAD_MEMORY must be default, host_pinned or host_mapped", .{}); + return error.InvalidArgument; + }; + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); + + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(init.environ_map)); + defer platform.deinit(allocator, io); + + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ + .checkpoint = path, + .read_parallelism = try envNumber(usize, init.environ_map, "ZML_LOAD_READ_PARALLELISM"), + .dma_block_size = if (dma_block_mib) |mib| @as(usize, mib) * zml.MiB else null, + .direct_io = direct_io, + }); + defer loader.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const tensor_parallel: *const zml.Mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + const model_axis: u3 = @intCast(tensor_parallel.resolveLogicalAxis("model").?); + const rows: zml.Sharding.Partitioning = .init(&.{.sharded(model_axis)}); + const columns: zml.Sharding.Partitioning = .init(&.{ .replicated, .sharded(model_axis) }); + + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); + defer store.deinit(); + + const Model = struct { tensors: []zml.Tensor }; + // The checkpoint tensor each one is bound to, at the same index. + const sources = store.registry.tensors.values(); + const model: Model = .{ .tensors = try allocator.alloc(zml.Tensor, sources.len) }; + defer allocator.free(model.tensors); + const view = store.view(); + for (model.tensors, store.registry.tensors.keys(), sources) |*tensor, key, source| { + const rank = source.shape.rank(); + const column_sharded = rank == 2 and + (std.mem.endsWith(u8, key, "o_proj.weight") or std.mem.endsWith(u8, key, "down_proj.weight")); + tensor.* = if (rank == 0 or mode == .replicated) + view.createReplicatedTensor(key, null) + else if (mode == .tp and column_sharded) + view.createTensor(key, null, .tp, columns) + else + view.createTensor(key, null, .tp, rows); + if (memory != .default) view.setMemory(tensor.*, memory); + } + + var progress = std.Progress.start(io, .{ .root_name = "load_check", .disable_printing = true }); + defer progress.end(); + + var buffers = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); + + if (check_stride != 0) try checkLoaded(allocator, io, sources, buffers.tensors, check_stride); +} + +/// Runs `dma-bench`: the DMA calibration screen on the platform, with the +/// `ZML_DMA_BENCH_*` knobs. +fn dmaBench(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const environ_map = init.environ_map; + + const defaults: zml.io.dma_calibration.Options = .{}; + const window_ms = try envNumber(i64, environ_map, "ZML_DMA_BENCH_WINDOW_MS"); + const opts: zml.io.dma_calibration.Options = .{ + .block_sizes = try envMibList(init.arena.allocator(), environ_map, "ZML_DMA_BENCH_BLOCK_MIB") orelse defaults.block_sizes, + .block_parallelism = try envNumber(usize, environ_map, "ZML_DMA_BENCH_BLOCK_PARALLELISM") orelse defaults.block_parallelism, + .minimum_duration = if (window_ms) |ms| .fromMilliseconds(ms) else defaults.minimum_duration, + .minimum_transfers = try envNumber(u64, environ_map, "ZML_DMA_BENCH_MIN_TRANSFERS") orelse defaults.minimum_transfers, + .block_selection_tolerance = try envNumber(f64, environ_map, "ZML_DMA_BENCH_BLOCK_TOLERANCE") orelse defaults.block_selection_tolerance, + }; + + const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(environ_map)); + defer platform.deinit(allocator, io); + + const block_size = try zml.io.dma_calibration.measure(allocator, io, platform, opts); + log.info("dma_benchmark block_bytes={d} parallelism={d}", .{ block_size, opts.block_parallelism }); +} + +/// The platform settings of the environment, which are not loader knobs. +fn platformOptions(environ_map: *const std.process.Environ.Map) !zml.platform.CreateOptions { + var options: zml.platform.CreateOptions = .{}; + if (try envNumber(u32, environ_map, "ZML_CPU_DEVICE_COUNT")) |device_count| { + options.cpu.device_count = device_count; + } + // Unset keeps the platform's allocator defaults (VMM on CUDA and ROCm); + // set, every XLA GPU target gets a preallocated BFC pool of that + // fraction, whose `bytes_limit` admission measures against. + if (try envNumber(f32, environ_map, "ZML_GPU_MEMORY_FRACTION")) |memory_fraction| { + const pool: zml.platform.CreateOptions.XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }; + options.cuda = pool; + options.rocm = pool; + options.oneapi = pool; + } + return options; +} + +/// Reads every `stride`-th of `buffers` and the largest eligible one back to +/// host and compares the bytes with the checkpoint tensor each was bound +/// to, `sources` at the same index, so a buffer that reported ready before +/// every piece landed is caught. A buffer every device holds whole is +/// compared on every replica; a partitioned one is assembled with +/// `toSliceAlloc`, which keeps the last replica of a region that several +/// devices hold. Each source file is opened once (an `hf://` open is a HEAD +/// plus a redirect). Skipped: sub-byte tensors partitioned over several +/// devices (`toSliceAlloc` places their shards by element stride). +fn checkLoaded( + allocator: std.mem.Allocator, + io: std.Io, + sources: []const zml.safetensors.Tensor, + buffers: []const zml.Buffer, + stride: usize, +) !void { + std.debug.assert(sources.len == buffers.len); + const Eligibility = struct { + fn checkable(source: zml.safetensors.Tensor, buffer: zml.Buffer) bool { + const partitioned = buffer.byteSize() / buffer.numShards() != source.byteSize(); + return !partitioned or source.shape.dtype().bitSizeOf() >= 8; + } + }; + var largest: ?usize = null; + for (sources, buffers, 0..) |source, buffer, i| { + if (!Eligibility.checkable(source, buffer)) continue; + if (largest == null or source.byteSize() > sources[largest.?].byteSize()) largest = i; + } + var files: std.StringHashMapUnmanaged(std.Io.File) = .empty; + defer { + var it = files.valueIterator(); + while (it.next()) |file| file.close(io); + files.deinit(allocator); + } + var expected: []u8 = &.{}; + defer allocator.free(expected); + var replica: []u8 = &.{}; + defer allocator.free(replica); + const started: std.Io.Timestamp = .now(io, .awake); + var checked: usize = 0; + var skipped: usize = 0; + var checked_bytes: usize = 0; + for (sources, buffers, 0..) |source, buffer, i| { + if (i % stride != 0 and i != largest) continue; + if (!Eligibility.checkable(source, buffer)) { + skipped += 1; + continue; + } + const file = files.get(source.file_uri) orelse blk: { + const opened = try std.Io.Dir.openFile(.cwd(), io, source.file_uri, .{ .mode = .read_only }); + errdefer opened.close(io); + try files.put(allocator, source.file_uri, opened); + break :blk opened; + }; + const source_bytes: usize = @intCast(source.byteSize()); + if (expected.len < source_bytes) { + allocator.free(expected); + expected = &.{}; + expected = try allocator.alloc(u8, source_bytes); + } + try source.readRange(io, file, expected[0..source_bytes], 0); + + if (buffer.byteSize() / buffer.numShards() == source_bytes) { + // Every shard holds the whole tensor: compare each replica. + if (replica.len < source_bytes) { + allocator.free(replica); + replica = &.{}; + replica = try allocator.alloc(u8, source_bytes); + } + var shards = buffer.shards(); + var shard_index: usize = 0; + while (shards.next()) |shard| : (shard_index += 1) { + try shard.toHost(io, replica[0..source_bytes]); + try expectSameBytes(source.name, shard_index, expected[0..source_bytes], replica[0..source_bytes]); + checked_bytes += source_bytes; + } + } else { + const slice = try buffer.toSliceAlloc(allocator, io); + defer slice.free(allocator); + try expectSameBytes(source.name, null, expected[0..source_bytes], slice.constData()); + checked_bytes += source_bytes; + } + checked += 1; + } + log.info("load check: ok tensors_checked={d} of {d} skipped={d} bytes={Bi:.2} elapsed={f}", .{ + checked, + sources.len, + skipped, + checked_bytes, + started.untilNow(io, .awake), + }); +} + +fn expectSameBytes(name: []const u8, replica: ?usize, expected: []const u8, actual: []const u8) !void { + if (actual.len == expected.len and std.mem.eql(u8, expected, actual)) return; + const first_bad = std.mem.indexOfDiff(u8, expected, actual) orelse @min(expected.len, actual.len); + log.err("load check: mismatch tensor={s} replica={?d} expected_bytes={d} actual_bytes={d} first_difference={d}", .{ + name, + replica, + expected.len, + actual.len, + first_bad, + }); + return error.LoadContentMismatch; +} + +/// The environment variable `name` as a comma-separated list of MiB, in +/// bytes, null when unset. An entry that does not parse is +/// `InvalidArgument`, logged. +fn envMibList(allocator: std.mem.Allocator, environ_map: *const std.process.Environ.Map, name: []const u8) !?[]const usize { + const text = environ_map.get(name) orelse return null; + var sizes: std.ArrayList(usize) = .empty; + errdefer sizes.deinit(allocator); + var entries = std.mem.tokenizeScalar(u8, text, ','); + while (entries.next()) |entry| { + const mib = std.fmt.parseInt(u32, entry, 10) catch { + log.err("{s} must be MiB counts separated by commas, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; + try sizes.append(allocator, @as(usize, mib) * zml.MiB); + } + return try sizes.toOwnedSlice(allocator); +} + +/// The environment variable `name` parsed as a `T`, null when unset. A value +/// that does not parse is `InvalidArgument`, logged. +fn envNumber(comptime T: type, environ_map: *const std.process.Environ.Map, name: []const u8) !?T { + const text = environ_map.get(name) orelse return null; + const value = switch (@typeInfo(T)) { + .int => std.fmt.parseInt(T, text, 10), + .float => std.fmt.parseFloat(T, text), + else => @compileError("envNumber: " ++ @typeName(T)), + }; + return value catch { + log.err("{s} must be a number, not '{s}'", .{ name, text }); + return error.InvalidArgument; + }; +} diff --git a/examples/io/main.zig b/examples/io/main.zig index c9dc05ea..e6132ff8 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -3,8 +3,6 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const log = std.log.scoped(.vfs); - pub const std_options: std.Options = .{ .log_level = .info, }; @@ -24,34 +22,12 @@ pub fn main(init: std.process.Init) !void { const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; const path = it.next() orelse return error.MissingPath; - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - try vfs.register("gs", gcs_vfs.io()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); @@ -132,9 +108,13 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - for (registry.tensors.values()) |tensor| { + var registry_it = registry.tensors.iterator(); + while (registry_it.next()) |kv| { + const name = kv.key_ptr.*; + const tensor = kv.value_ptr.*; + var current = root; - var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); + var parts = std.mem.tokenizeScalar(u8, name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -158,6 +138,11 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Returns at once: the loader sets its backend up while the + // checkpoint's headers are read and its tensors bound. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); + defer loader.deinit(); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); @@ -167,58 +152,36 @@ pub fn main(init: std.process.Init) !void { ); const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); + // The store takes the registry's entries. + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); defer store.deinit(); const AllTensorsModel = struct { tensors: []zml.Tensor, }; - const tensor_count = registry.tensors.count(); - - const tensors = try allocator.alloc(zml.Tensor, tensor_count); + const tensors = try allocator.alloc(zml.Tensor, store.registry.tensors.count()); defer allocator.free(tensors); - var load_count: usize = 0; - for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { - tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(name, null), - .sharded => if (tensor.shape.rank() > 0) - store.view().createTensor(name, null, .tp, tp_partitioning) + for (tensors, store.registry.tensors.keys(), store.registry.tensors.values()) |*tensor, key, entry| { + tensor.* = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(key, null), + .sharded => if (entry.shape.rank() > 0) + store.view().createTensor(key, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(name, null), + store.view().createReplicatedTensor(key, null), }; - load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); - progress.increaseEstimatedTotalItems(load_count); defer progress.end(); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); - defer { - for (buffers.tensors) |*b| b.deinit(); - allocator.free(buffers.tensors); - } - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .parallelism = 8, - .dma_chunks = 16, - .dma_chunk_size = 256 * zml.MiB, - }); - defer loader.deinit(); - - try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + var buffers = try zml.mem.bufferize(allocator, &model); + defer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &store, progress); + try loader.wait(); // logs "Loaded weights" }, } } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index b24c022a..b5c8f3ec 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -53,28 +53,12 @@ pub fn main(init: std.process.Init) !void { // // Virtual File Systems // - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer http_client.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("gs", gcs_vfs.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); + var remotes: zml.io.VFS.Remotes = undefined; + try remotes.init(allocator, &vfs, init.environ_map); + defer remotes.deinit(); const io = vfs.io(); @@ -107,7 +91,7 @@ pub fn main(init: std.process.Init) !void { // Defines how the model's tensors are sharded across the available devices. const meshes: models.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -116,9 +100,6 @@ pub fn main(init: std.process.Init) !void { }, }; - var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); - defer model.deinit(allocator); - // // Load the model and compile it // @@ -128,14 +109,29 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); + var model: models.LoadedModel = undefined; var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); - defer compiled_model.deinit(); + var model_buffers = buffers: { + // Returns at once: the loader sets its backend up on a task of its + // own while the model is initialized and compiled. It is freed at + // the end of this block, with its pinned host memory, before the + // chat starts. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = args.model }); + defer loader.deinit(); - // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); - defer model.unloadBuffers(&model_buffers, allocator); + model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + errdefer model.deinit(allocator); + + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); + errdefer compiled_model.deinit(); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + break :buffers try models.LoadedModel.loadBuffers(&model, allocator, &loader, progress, &store); + }; + defer model.deinit(allocator); + defer compiled_model.deinit(); + defer zml.mem.deinitBufferized(allocator, &model_buffers); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 20aa60c7..33e22bf3 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -55,26 +55,23 @@ pub const LoadedModel = union(ModelType) { } } + /// Loads the model's weights from the store with `loader`, which logs + /// them. Free them with `zml.mem.deinitBufferized`. pub fn loadBuffers( - self: *LoadedModel, + self: *const LoadedModel, allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, + loader: *zml.io.Loader, + progress: std.Progress.Node, + store: *const zml.io.TensorStore, ) !Buffers { - return switch (self.*) { - inline else => |*m, t| @unionInit( - Buffers, - @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress), - ), - }; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { switch (self.*) { - inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), + inline else => |*m, t| { + var buffers = try zml.mem.bufferize(allocator, &m.inner); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + try loader.start(&m.inner, &buffers, store, progress); + try loader.wait(); + return @unionInit(Buffers, @tagName(t), buffers); + }, } } diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 9f7cb909..bee197fa 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.lfm); - pub const Config = struct { architectures: []const []const u8, block_auto_adjust_ff_dim: bool, @@ -72,43 +70,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - Model.unloadBuffers(buffers, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -164,47 +125,6 @@ pub const Model = struct { }; } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 8, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - LmHead.unloadBuffers(&self.lm_head); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - pub fn forward( self: Model, tokens: zml.Tensor, @@ -324,10 +244,6 @@ pub const LmHead = struct { const tokens = new_tokens.convert(.u32); return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; } - - pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { - RmsNorm.unloadBuffers(&self.embedding_norm); - } }; pub const OperatorKind = enum { conv, full_attention }; @@ -426,16 +342,6 @@ pub const DecoderLayer = struct { .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), }; } - - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - switch (self.operator) { - .conv => |*operator| ShortConv.unloadBuffers(operator), - .self_attn => |*operator| Attention.unloadBuffers(operator), - } - RmsNorm.unloadBuffers(&self.operator_norm); - RmsNorm.unloadBuffers(&self.ffn_norm); - Mlp.unloadBuffers(&self.feed_forward); - } }; pub const ConvParameters = struct { is_prefill: bool }; @@ -501,12 +407,6 @@ pub const ShortConv = struct { const output = self.out_proj.forward(y); return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { - Linear.unloadBuffers(&self.in_proj); - Linear.unloadBuffers(&self.out_proj); - self.kernel.deinit(); - } }; pub const Attention = struct { @@ -571,8 +471,6 @@ pub const Attention = struct { v = new_kv_cache.values(cache_index); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); - const mesh = zml.Compiler.current().mesh(.model); - const partitioning: zml.attention.Partitioning = .head(.model); const attn = switch (attention_parameters) { .stablehlo => zml.attention.attention( q, @@ -581,8 +479,6 @@ pub const Attention = struct { tokens_position_offset, attention_metadata, attention_parameters, - mesh, - partitioning, ).merge(.{ .d = .{ .h, .hd } }), else => zml.attention.attention( q.squeeze(.batch), @@ -591,8 +487,6 @@ pub const Attention = struct { tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters, - mesh, - partitioning, ).merge(.{ .d = .{ .h, .hd } }) .rename(.{ .q = .seq }) .insertAxes(.seq, .{.batch}), @@ -600,15 +494,6 @@ pub const Attention = struct { return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; } - - pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { - Linear.unloadBuffers(&self.q_proj); - Linear.unloadBuffers(&self.k_proj); - Linear.unloadBuffers(&self.v_proj); - Linear.unloadBuffers(&self.out_proj); - RmsNorm.unloadBuffers(&self.q_layernorm); - RmsNorm.unloadBuffers(&self.k_layernorm); - } }; pub const Cache = struct { @@ -619,11 +504,6 @@ pub const Cache = struct { return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } - pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { - ConvCache.unloadBuffers(&self.conv); - KvCache.unloadBuffers(&self.kv); - } - pub fn reuseBuffer(self: Cache, other: Cache) Cache { return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; } @@ -644,10 +524,6 @@ pub const ConvCache = struct { return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } - pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { - self.state.deinit(); - } - pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { return .{ .state = self.state.reuseBuffer(other.state) }; } @@ -666,11 +542,6 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } - pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - } - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; } @@ -706,11 +577,6 @@ pub const Linear = struct { return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; } - pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { - linear.weight.deinit(); - if (linear.bias) |*bias| bias.deinit(); - } - pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; @@ -730,12 +596,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); } @@ -750,10 +610,6 @@ const RmsNorm = struct { return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index ecc5abf8..df617bec 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -31,7 +31,7 @@ pub const Session = struct { ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); - errdefer model.Cache.unloadBuffers(&cache_buffers); + errdefer zml.mem.deinitBufferized(allocator, &cache_buffers); var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); @@ -62,7 +62,7 @@ pub const Session = struct { pub fn deinit(self: *Session) void { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); - model.Cache.unloadBuffers(&self.cache_buffers); + zml.mem.deinitBufferized(self.allocator, &self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 19b28df6..4934d3e6 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -41,7 +41,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -49,8 +49,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { .stablehlo => .{ .stablehlo = {} }, @@ -85,7 +89,7 @@ pub fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer activation_store.deinit(); var ctx = TestContext{ @@ -175,7 +179,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(self.io, args, &res, .{}); + try exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -304,16 +308,12 @@ const TestContext = struct { }; fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { - const shape = store.view().getShape(key) orelse return error.NotFound; + const tensor = store.view().get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.view().getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 8a6a09e9..0dfcbb18 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.llama); - pub const Config = struct { bos_token_id: u32, eos_token_id: EosTokens, @@ -69,43 +67,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&buffers.model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -161,43 +122,6 @@ pub const Model = struct { self.model.deinit(allocator); } - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&self.model, allocator); - } - fn lmHead(self: Model) LmHead { return .init(self); } @@ -270,15 +194,6 @@ const Llama = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - RmsNorm.unloadBuffers(&self.norm); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward( @@ -329,7 +244,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .replicate() }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -396,13 +311,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - SelfAttn.unloadBuffers(&self.self_attn); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - Mlp.unloadBuffers(&self.mlp); - } - pub const Input = struct { layer: TransformerLayer, hidden: zml.Tensor, @@ -461,10 +369,6 @@ const RmsNorm = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -486,10 +390,6 @@ const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const proj = self.up_proj.forward(x, x.dtype()); var output = self.gate_proj.forward(x, x.dtype()); @@ -531,10 +431,6 @@ const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - /// Self zml.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. @@ -587,8 +483,6 @@ const SelfAttn = struct { token_index, attention_metadata, attention_parameters, - zml.Compiler.current().mesh(.model), - .head(.model), ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 1b5894f1..dae0d054 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -47,8 +47,12 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); + var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); + defer loader.deinit(); + var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); + defer zml.mem.deinitBufferized(allocator, &model_buffers); + try loader.start(&repo_model.inner, &model_buffers, &store, progress); + try loader.wait(); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); @@ -66,7 +70,7 @@ fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); + var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = meshes }); defer activation_store.deinit(); try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); @@ -98,7 +102,7 @@ fn testLayer( ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); - const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + const in_shape = (activation_store.get(in_key) orelse return error.NotFound).shape; var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); @@ -126,7 +130,7 @@ fn testLayer( var res = try exe.results(allocator); defer res.deinit(allocator); - exe.call(io, args, &res, .{}); + try exe.call(io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -134,16 +138,12 @@ fn testLayer( } fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { - const shape = store.getShape(key) orelse return error.NotFound; + const tensor = store.get(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, shape.byteSize()); + const host_bytes = try allocator.alloc(u8, tensor.byteSize()); defer allocator.free(host_bytes); - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.getReader(key, io, &io_buffer); - defer reader.deinit(); + try tensor.read(io, host_bytes); - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 40174f5c..a434c489 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -6,8 +6,6 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.qwen3_5); - pub const Config = struct { text_config: TextConfig, }; @@ -74,44 +72,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - zml.nn.Linear.unloadBuffers(&buffers.lm_head); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -153,7 +113,7 @@ pub const Model = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer const lm_head_prefix = b: { - if (store.hasKey("lm_head.weight")) break :b "lm_head"; + if (store.get("lm_head.weight") != null) break :b "lm_head"; break :b "model.language_model.embed_tokens"; }; return .{ @@ -172,43 +132,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -290,7 +213,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .replicate() }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -322,15 +245,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - } - pub fn forward( self: TextModel, tokens: zml.Tensor, @@ -401,16 +315,6 @@ pub const TransformerLayer = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Mlp.unloadBuffers(&self.mlp); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forward( self: TransformerLayer, x0: zml.Tensor, @@ -522,10 +426,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -577,10 +477,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); @@ -657,10 +553,6 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); - const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { - .sharded => .head(.model), - .replicated => .head(.replicated), - }; const attn_output = zml.attention.attention( q, k, @@ -668,8 +560,6 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), - zml.Compiler.current().mesh(.model), - head_partitioning, ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); @@ -804,10 +694,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -983,10 +869,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -1004,10 +886,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 2a3b0ba8..9fa5300f 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -100,43 +100,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn loadBuffers( - self: *LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -188,42 +151,6 @@ pub const Model = struct { self.text_model.deinit(allocator); } - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -305,7 +232,7 @@ pub const TextModel = struct { config: Config, gen_options: Model.GenOptions, ) !TextModel { - const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + const lm_head_prefix = if (store.root().get("lm_head.weight") != null) "lm_head" else "model.language_model.embed_tokens"; const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); errdefer allocator.free(layers); @@ -331,16 +258,6 @@ pub const TextModel = struct { allocator.free(self.layers); } - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer, allocator); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } - pub fn sampler(self: TextModel) Sampler { return .{ .norm = self.norm, @@ -437,16 +354,6 @@ pub const TransformerLayer = struct { self.moe.deinit(allocator); } - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Moe.unloadBuffers(&self.moe, allocator); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; @@ -582,10 +489,6 @@ pub const SelfAttn = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); @@ -641,10 +544,6 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); - const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { - .sharded => .head(.model), - .replicated => .head(.replicated), - }; const attn_output = zml.attention.attention( q, k, @@ -652,9 +551,7 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), - zml.Compiler.current().mesh(.model), - head_partitioning, - ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); @@ -694,10 +591,6 @@ pub const Mlp = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -715,18 +608,14 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{}), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{}), .d, ), .num_experts_per_tok = num_experts_per_tok, }; } - pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { - zml.Buffer.deinitAll(Router, self); - } - pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const router_logits = self.router.forward(x, .f32); const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); @@ -762,8 +651,8 @@ pub const Moe = struct { return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{}), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{}), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -787,8 +676,6 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, moe_backend, - zml.Compiler.current().mesh(.experts), - .expert(.experts), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, @@ -805,11 +692,6 @@ pub const Moe = struct { return moe_output.add(shared); } - - pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { - _ = allocator; - zml.Buffer.deinitAll(Moe, self); - } }; pub const TextRotaryEmbedding = struct { @@ -935,10 +817,6 @@ pub const GatedDeltaNet = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -1104,10 +982,6 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -1125,10 +999,6 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f5842af8..878b269b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,32 +37,6 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer unloadBuffers(&buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -80,24 +54,32 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); defer store.deinit(); + + // Init model const mnist_model: Mnist = .init(store.view()); // // Compile model @@ -110,16 +92,11 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer Mnist.unloadBuffers(&mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); @@ -147,7 +124,7 @@ pub fn main(init: std.process.Init) !void { printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index b7805e49..76684d94 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -2,11 +2,9 @@ //! Separated to kept `mnist.zig` minimal const std = @import("std"); - const zml = @import("zml"); const base = @import("mnist.zig"); - const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ @@ -33,42 +31,17 @@ const Mnist = struct { }; } - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer zml.Buffer.deinitAll(Mnist, &buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); - const layers: []const Layer = &.{ self.fc1, self.fc2 }; - const compiler = zml.Compiler.current(); - for (0.., layers) |i, layer| { compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); defer compiler.popLocation(); x = layer.forward(x); } - - return x.argMax(.d).indices.convert(.u8) - .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) - .toMemory(.host_pinned); + return x.argMax(.d).indices.convert(.u8).withPartitioning(input.shape()._sharding.mesh.?, .{ .b = .data }).toMemory(.host_pinned); } const Layer = struct { @@ -76,7 +49,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + log.warn("layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); const x = input.dot(self.weight, .d); return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); } @@ -86,16 +59,16 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; - const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); + // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + const io = vfs.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); @@ -108,20 +81,25 @@ pub fn main(init: std.process.Init) !void { &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }, )); - log.info("{f}", .{dp_mp}); + log.info("topology used: {f}", .{dp_mp}); + + // Returns at once: the loader calibrates while the model is compiled. + var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); + defer loader.deinit(); var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{dp_mp} }); defer store.deinit(); - const mnist_model: Mnist = .init(store.view()); - // Compile model + // // Compile model const bs: u32 = 4; const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); - var exe = exe: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); @@ -135,22 +113,17 @@ pub fn main(init: std.process.Init) !void { .{ .program_name = "mnist", .meshes = &.{dp_mp}, - .xla_dump_to = "/tmp/zml/mnist_dp", + .xla_dump_to = "/tmp/zml/mnist", }, ); }; defer exe.deinit(); - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + // Load weights + var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); + defer zml.mem.deinitBufferized(allocator, &mnist_buffers); + try loader.start(&mnist_model, &mnist_buffers, &store, .none); + try loader.wait(); // logs "Loaded weights" var args = try exe.args(allocator); defer args.deinit(allocator); @@ -186,7 +159,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ mnist_buffers, input_buffer }); try profiler.start(); - exe.call(io, args, &results, .{}); + try exe.call(io, args, &results, .{}); if (try profiler.stop()) |report| { log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); } diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 34a78bfe..29de9597 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -30,18 +30,14 @@ const DemoModel = struct { } pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { - const compiler = zml.Compiler.current(); - const x = input.convert(.f32); + var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid(); - - const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - + const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); return zml.ops.manualComputation( (struct { y: zml.Tensor, @@ -54,11 +50,6 @@ const DemoModel = struct { }).body, .{ .y = y, .gate = gate }, y.shape(), - compiler.mesh(.dp_mp), - .{ - .in_specs = .{ .y = spec, .gate = spec }, - .out_specs = spec, - }, ); } }; @@ -69,7 +60,7 @@ fn usage() void { \\ \\Simple sharding demo: \\ - explicit partitioning on data/model axes - \\ - tensor print + \\ - shard-local tensor print \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: @@ -251,7 +242,7 @@ pub fn main(init: std.process.Init) !void { defer exe_results.deinit(allocator); exe_args.set(.{ w_buf, b_buf, input_buf }); - exe.call(io, exe_args, &exe_results, .{}); + try exe.call(io, exe_args, &exe_results, .{}); var out = exe_results.get(zml.Buffer); defer out.deinit(); diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc index 6e7c296d..fe1296db 100644 --- a/mlir/dialects/cute_ir/capi.cc +++ b/mlir/dialects/cute_ir/capi.cc @@ -166,10 +166,6 @@ MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { return getAlgebraValue(attr); } -intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { - return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); -} - MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { return getStaticAttr(context, value); } diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h index 002f4885..e9e285d2 100644 --- a/mlir/dialects/cute_ir/capi.h +++ b/mlir/dialects/cute_ir/capi.h @@ -54,7 +54,6 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( MlirContext context, intptr_t numElements, MlirAttribute const *elements); MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); -MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index b277d061..8a031d0b 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -145,10 +145,6 @@ pub const ShapeAttr = opaque { return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); } - pub fn rank(self: *const ShapeAttr) usize { - return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); - } - pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { return @ptrCast(self); } diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index 9673870f..b66b7abe 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index d1197317..78524e0a 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -22,7 +22,6 @@ pub const elementtype = ops.elementtype; pub const AtomicBinOp = ops.AtomicBinOp; pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; -pub const ptrtoint = ops.ptrtoint; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; pub const extractvalue = ops.extractvalue; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index d1fb5771..2102f68a 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -234,15 +234,6 @@ pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *co }); } -/// llvm.ptrtoint — the integer address of an LLVM pointer. -pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ - .operands = .{ .flat = &.{pointer} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - /// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "llvm.bitcast", .{ diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index 6ba84b4d..28541d44 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -20,7 +20,6 @@ const operation_names: []const []const u8 = &.{ "llvm.intr.lifetime.end", "llvm.intr.lifetime.start", "llvm.inttoptr", - "llvm.ptrtoint", "llvm.load", "llvm.mlir.poison", "llvm.mlir.undef", diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig index 14a0e8e2..d46f3a4f 100644 --- a/mlir/dialects/nvvm/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -136,8 +136,7 @@ pub fn mbarrier_wait_parity( } /// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) -/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) -/// is a suspend-time hint in nanoseconds per try. +/// of the mbarrier completes, suspending up to `suspend_time` (i32) cycles per try. pub fn mbarrier_try_wait_parity( ctx: *mlir.Context, ptr: *const mlir.Value, @@ -148,27 +147,6 @@ pub fn mbarrier_try_wait_parity( return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); } -/// One potentially suspending parity test with acquire semantics. A false -/// result means the timeout expired; callers that need completion must retry. -pub fn mbarrier_try_wait_parity_timelimit( - ctx: *mlir.Context, - ptr: *const mlir.Value, - parity: *const mlir.Value, - suspend_time: *const mlir.Value, - scope: MBarrierScope, - location: *const mlir.Location, -) *mlir.Operation { - return ops.mbarrier_try_wait_parity_timelimit( - ctx, - ptr, - parity, - suspend_time, - .int(ctx, .i1), - .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, - location, - ); -} - /// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return ops.fence_mbarrier_init(ctx, location); @@ -230,12 +208,10 @@ pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the -/// tcgen05 MMA/copy operations issued by the executing thread with `group` -/// complete. An optional mask selects CTAs in the cluster whose barriers at -/// the same shared-memory offset receive one arrival each. -pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued +/// tcgen05 operations complete. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); } /// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..499e753c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -484,11 +484,6 @@ pub const Client = opaque { return @ptrCast(ret.executable.?); } - pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { - var future = io.async(compileRaw, .{ self, api, args }); - return future.await(io); - } - pub const BufferFromHostBufferArgs = struct { data: [*]const u8, buffer_type: BufferType, @@ -677,8 +672,19 @@ pub const Client = opaque { }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.call); + } + + /// As `createUninitializedBuffer`, for a request the caller expects to + /// fail: its error is logged at debug level only. + pub fn createUninitializedBufferSilent(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return self.createUninitializedBufferThrough(api, args, Api.callSilent); + } + + /// `call` is `Api.call` or `Api.callSilent`. + fn createUninitializedBufferThrough(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs, comptime call: anytype) ApiError!*Buffer { var layout = args.layout.toCStruct(); - const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ + const ret = try call(api, .PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 08817afe..1105f776 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8482e52a..ced8ae98 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", + "sha256": "77d2ddb19566604c827828fe3dcc53e938f3a1b16d431416fdd5c466266f5242", + "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", + "sha256": "2cd18393c44816d3d01aa22fbd890632ba46eca3939c644bddef00d590578a80", + "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index afb26864..0aa3905d 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", - sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", + url = "https://mirror.zml.ai/plugins/202610221403.283.1.d9f4f68759e1/zml-furiosa-linux-amd64.tar.zst", + sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ee33fd58..ebbd9e3c 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,6 +50,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 71e1858a..03cdaf02 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", + sha256 = "6eee91776397d472ac590413cf44d4ff3f7b31bebb163aa96339c6becaf2defd", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index a3c968d3..6016e389 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" ONEAPI_VERSION = "2026.1" @@ -248,7 +248,6 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", - "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 140ce102..a3e69ad4 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 5a829ac5..272bc02c 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,11 +15,9 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics - # packages are pinned below to keep their versions from it. Use snapshots, - # not the live PPA: the PPA deletes superseded packages from its pool. + # IGC 2.41.15 is newer than the graphics snapshot above. - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z + url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu archs: - "amd64" @@ -45,9 +43,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" + - "libigdgmm12" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" + - "libze-intel-gpu1" - "libze1" - "zlib1g" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 41b01b60..babd73ab 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cd6c3c8673ce690a0fbf89dab5a16e6893f0f501bb2b41997c657cf6effa7549" def _rocm_package_name(name): return name + _ROCM_VERSION @@ -73,7 +73,6 @@ def _rocm_runfile_build_files(loaded_packages): for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ - ".kpack/**", "lib/libMIOpenCKGroupedConv_*.so", "share/miopen/db/**", ]) diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index ee31bd83..6fcda16b 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -74,6 +74,142 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); - break :blk .loadFrom(lib_path); + const api = try pjrt.Api.loadFrom(lib_path); + loaded_hip = Hip.loaded(); + break :blk api; }; } + +/// Whether `releaseRuntimeDeviceMemory` asks the devices for their whole +/// memory. The first call that finds the HIP runtime missing, or a device +/// that does not refuse, logs it and clears this for good. It lives here, +/// not with a PJRT client: the HIP runtime the plugin loaded, and the +/// memory it keeps, belong to the process and serve every client. +var releases_runtime_memory: std.atomic.Value(bool) = .init(true); +/// The HIP calls of `releaseRuntimeDeviceMemory`, resolved by `load`, which +/// zml calls once per process; null when `libamdhip64.so.7` is not loaded +/// or lacks one of them. +var loaded_hip: ?Hip = null; + +/// Makes the HIP runtime the plugin loaded give back the device memory it +/// keeps outside the plugin's allocator, chiefly the scratch of kernels +/// that spill registers, which ROCr keeps bound to its queues and in its +/// caches until the driver refuses one of its allocations. It asks every +/// device in turn for its whole memory, rounded down to 2 MiB: the driver +/// refuses, ROCr reclaims the scratch of the device's queues and frees its +/// caches, and HIP answers `hipErrorOutOfMemory`. Kernels that spill later +/// map their scratch again. Every queue in the process that holds scratch +/// on the device is suspended and resumed, and the call waits for its +/// kernels using scratch while holding the device's ROCr memory lock: +/// running work and allocations on that device stall meanwhile, a stall, +/// not a deadlock. Restores the thread's HIP device, and clears any HIP +/// error left on the thread, earlier ones included. +/// +/// Returns whether every device refused. A missing `libamdhip64.so.7` or +/// symbol, or a device that does not refuse (a grant is freed at once), is +/// logged once and ends the trim for good: the remaining devices are not +/// asked, nor any device in later calls. +pub fn releaseRuntimeDeviceMemory(hip_ordinals: []const c_int) bool { + if (comptime !isEnabled() or builtin.os.tag != .linux) return false; + if (!releases_runtime_memory.load(.monotonic)) return false; + const hip = loaded_hip orelse { + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("The HIP runtime (libamdhip64.so.7) is not loaded or lacks a symbol; the device memory it keeps is no longer given back", .{}); + } + return false; + }; + + var previous: c_int = 0; + const has_previous = hip.hipGetDevice(&previous) == hip_success; + defer { + if (has_previous) _ = hip.hipSetDevice(previous); + _ = hip.hipGetLastError(); + } + + for (hip_ordinals) |ordinal| { + const answer = hip.requestWholeDevice(ordinal); + if (answer == .refused) continue; + if (releases_runtime_memory.swap(false, .monotonic)) { + log.warn("HIP device {d} did not refuse a request for its whole memory: {f}; the device memory the HIP runtime keeps is no longer given back", .{ ordinal, answer }); + } + return false; + } + return true; +} + +/// `hipError_t` values (hip_runtime_api.h). +const hip_success: c_int = 0; +const hip_error_out_of_memory: c_int = 2; + +/// The HIP calls `releaseRuntimeDeviceMemory` makes, from the HIP runtime +/// already in the process. +const Hip = struct { + hipGetDevice: *const fn (device: *c_int) callconv(.c) c_int, + hipSetDevice: *const fn (device: c_int) callconv(.c) c_int, + hipMemGetInfo: *const fn (free: *usize, total: *usize) callconv(.c) c_int, + hipMalloc: *const fn (ptr: *?*anyopaque, size: usize) callconv(.c) c_int, + hipFree: *const fn (ptr: ?*anyopaque) callconv(.c) c_int, + hipGetLastError: *const fn () callconv(.c) c_int, + + /// Only finds the library: it never loads it, nor unloads it (see the + /// note on thread-local storage in `load`). `libzml_rocm.so` needs + /// `libamdhip64.so.7`, and the plugin's own dlopen of `libamdhip64.so` + /// resolves to the same file (`zmlxrocm_dlopen`). + fn loaded() ?Hip { + var lib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libamdhip64.so.7", .{ .LAZY = true, .NOLOAD = true }) orelse return null, + } }; + var hip: Hip = undefined; + inline for (@typeInfo(Hip).@"struct".field_names) |name| { + @field(hip, name) = lib.lookup(@FieldType(Hip, name), name) orelse return null; + } + return hip; + } + + /// Asks HIP for the whole memory of the device, made the current one, + /// when less than that is free, so that the driver cannot grant it. + /// Frees what HIP grants anyway. + fn requestWholeDevice(hip: Hip, ordinal: c_int) Answer { + const set_device = hip.hipSetDevice(ordinal); + if (set_device != hip_success) return .{ .set_device = set_device }; + var free: usize = 0; + var total: usize = 0; + const mem_get_info = hip.hipMemGetInfo(&free, &total); + if (mem_get_info != hip_success) return .{ .mem_get_info = mem_get_info }; + // ROCr rounds a request up to 2 MiB and refuses one beyond the + // device's size itself, without the driver's refusal that trims. + const request = std.mem.alignBackward(usize, total, 2 << 20); + if (free >= request) return .{ .not_asked = .{ .free = free, .request = request } }; + var ptr: ?*anyopaque = null; + const answer = hip.hipMalloc(&ptr, request); + if (answer == hip_error_out_of_memory) return .refused; + if (answer == hip_success) _ = hip.hipFree(ptr); + return .{ .malloc = answer }; + } +}; + +/// How a device answered `Hip.requestWholeDevice`. +const Answer = union(enum) { + /// hipMalloc answered `hipErrorOutOfMemory`, as expected. + refused, + /// So much is free that the driver could grant the request. + not_asked: struct { free: usize, request: usize }, + /// hipMalloc's other answers: `hip_success` for a grant, freed at once. + malloc: c_int, + /// The `hipError_t` of hipSetDevice or hipMemGetInfo, which failed. + set_device: c_int, + mem_get_info: c_int, + + pub fn format(answer: Answer, writer: *std.Io.Writer) std.Io.Writer.Error!void { + switch (answer) { + .refused => try writer.writeAll("refused"), + .not_asked => |memory| try writer.print("not asked, {d} bytes being free for a request of {d}", .{ memory.free, memory.request }), + .malloc => |code| if (code == hip_success) + try writer.writeAll("granted, then freed") + else + try writer.print("hipMalloc returned hipError_t {d}", .{code}), + .set_device => |code| try writer.print("hipSetDevice returned hipError_t {d}", .{code}), + .mem_get_info => |code| try writer.print("hipMemGetInfo returned hipError_t {d}", .{code}), + } + } +}; diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 6a0fdfae..4c1c50ed 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/stdx/Io.zig b/stdx/Io.zig index 5b5a35d8..453ee503 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -22,130 +22,20 @@ pub const Dir = struct { } }; -pub const LimitedGroup = struct { - limit: usize, - in_flight: std.atomic.Value(usize) = .init(0), - group: std.Io.Group = .init, - cond: std.Io.Condition = .init, - mutex: std.Io.Mutex = .init, - - fn Wrapper(comptime function: anytype) type { - return struct { - fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.limit) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - defer { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.cond.signal(io); - } - return @call(.auto, function, args); - } - - try self.mutex.lock(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.limit) { - try self.cond.wait(io, &self.mutex); - } - } - } - }; - } - - pub fn init(limit: usize) LimitedGroup { - return .{ .limit = limit }; - } - - pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { - self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { - try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); - } - - pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *LimitedGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - -pub const AllocatingLimitedConcurrentGroup = struct { - allocator: std.mem.Allocator, - - queue_buffer: []bool = &[_]bool{}, - limit: usize, - - group: std.Io.Group = .init, - queue: std.Io.Queue(bool) = undefined, - - pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { - const buffer = try allocator.alloc(bool, limit); - @memset(buffer, false); - var self: AllocatingLimitedConcurrentGroup = .{ - .allocator = allocator, - .queue_buffer = buffer, - .limit = limit, - .group = .init, - .queue = undefined, - }; - self.queue = std.Io.Queue(bool).init(self.queue_buffer); - return self; - } - - pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { - self.allocator.free(self.queue_buffer); - } - - pub fn concurrent( - self: *AllocatingLimitedConcurrentGroup, - io: std.Io, - comptime function: anytype, - args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - self.queue.putOneUncancelable(io, true) catch unreachable; - - try self.group.concurrent(io, struct { - fn wrapper( - io_: std.Io, - queue: *std.Io.Queue(bool), - inner_args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - defer _ = queue.getOneUncancelable(io_) catch unreachable; - return @call(.auto, function, inner_args); - } - }.wrapper, .{ io, &self.queue, args }); - } - - pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } - - pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { - self.group.cancel(io); - } -}; - pub const Duration = struct { pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); } pub fn hz(self: std.Io.Duration) u64 { - return @intCast(std.time.ns_per_s / self.nanoseconds); + return @intCast(@divTrunc(std.time.ns_per_s, self.nanoseconds)); } pub fn hzFloat(self: std.Io.Duration) f64 { return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); } }; + +test "Duration.hz" { + try std.testing.expectEqual(@as(u64, 4), Duration.hz(.fromMilliseconds(250))); +} diff --git a/stdx/flags.zig b/stdx/flags.zig index d0c4352d..52a9e6a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,19 +591,25 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } -/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - comptime assert(!std.mem.eql(u8, field_name, "positional")); - comptime var name = field_name[0..field_name.len].*; - comptime std.mem.replaceScalar(u8, &name, '_', '-'); - const dashed = name; - return "--" ++ dashed; + // TODO(Zig): Cleanup when this is fixed after Zig 0.11. + // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. + return comptime blk: { + assert(!std.mem.eql(u8, field_name, "positional")); + + var result: []const u8 = "--"; + var index = 0; + while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { + result = result ++ field_name[index..][0..i] ++ "-"; + index = index + i + 1; + } + result = result ++ field_name[index..]; + break :blk result; + }; } test flag_name { - try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); - try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); - try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); + try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 1b19e638..5ed0b048 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -87,6 +87,33 @@ pub fn Once(comptime f: anytype) type { }; } +/// The first error of concurrent tasks, kept once and readable by any of +/// them. The state is the error's integer value, zero while none is kept. +pub fn FirstError(comptime E: type) type { + return struct { + code: std.atomic.Value(@Int(.unsigned, @bitSizeOf(anyerror))) = .init(0), + + /// Keeps `err` unless an error is kept already; true when this call + /// kept it. + pub fn set(self: *@This(), err: E) bool { + return self.code.cmpxchgStrong(0, @intFromError(err), .release, .monotonic) == null; + } + + pub fn get(self: *const @This()) ?E { + const code = self.code.load(.acquire); + return if (code == 0) null else @errorCast(@errorFromInt(code)); + } + }; +} + +test FirstError { + var first: FirstError(error{ A, B }) = .{}; + try std.testing.expectEqual(null, first.get()); + try std.testing.expect(first.set(error.B)); + try std.testing.expect(!first.set(error.A)); + try std.testing.expectEqual(error.B, first.get().?); +} + /// Packed storage of sub-byte values /// /// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 69cc6128..d53e20be 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -30,7 +30,7 @@ zls_toolchain( toolchain( name = "toolchain", - target_settings = ["@zig_toolchains//:any_0.17.release"], + target_settings = ["@zig_toolchains//:0.17.0"], toolchain = ":zls", toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel deleted file mode 100644 index aff4954d..00000000 --- a/tools/safetensors/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "safetensors", - main = "main.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - "//stdx", - "//vfs", - "//zml", - ], -) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig deleted file mode 100644 index cf902eef..00000000 --- a/tools/safetensors/main.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const VFS = @import("vfs"); -const zml = @import("zml"); - -const log = std.log.scoped(.safetensors_convert); - -const Args = struct { - output: ?[]const u8 = null, - positional: struct { model: []const u8 }, - - pub const help = - \\Usage: [--output=] - \\ - \\Read safetensors headers and write model.safetensors.full_index.json in the working directory - \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash - ; -}; - -pub fn main(init: std.process.Init) !void { - const args = stdx.flags.parse(init.minimal.args, Args); - const allocator = init.gpa; - - const start = std.Io.Clock.now(.awake, init.io); - var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer client.deinit(); - try client.initDefaultProxies(allocator, init.environ_map); - - const input = args.positional.model; - - var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); - defer file_vfs.deinit(); - var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); - defer http_vfs.deinit(); - var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); - defer https_vfs.deinit(); - var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (hf_vfs) |*backend| backend.deinit(); - var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (s3_vfs) |*backend| backend.deinit(); - var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (gcs_vfs) |*backend| backend.deinit(); - - var vfs: VFS = try .init(allocator, init.io); - const io = vfs.io(); - defer vfs.deinit(); - try vfs.register("file", file_vfs.io()); - try vfs.register("http", http_vfs.io()); - try vfs.register("https", https_vfs.io()); - if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); - if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); - if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); - - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const repo = try zml.safetensors.resolveModelRepo(io, input); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const loaded = std.Io.Clock.now(.awake, init.io); - - const index_json_file = if (args.output) |output| - try std.Io.Dir.cwd().createFile(io, output, .{}) - else - try repo.createFile(io, full_index_path, .{}); - const out_buffer = try allocator.alloc(u8, 4096); - defer allocator.free(out_buffer); - var out_writer = index_json_file.writer(io, out_buffer); - try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); - const done = std.Io.Clock.now(.awake, init.io); - - log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ - args.output orelse full_index_path, - registry.tensors.count(), - start.durationTo(loaded), - loaded.durationTo(done), - start.durationTo(done), - }); -} - -const full_index_path = "model.safetensors.full_index.json"; diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index c8de1e0c..352ce826 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -4,11 +4,14 @@ zig_library( name = "vfs", srcs = [ "base.zig", - "file.zig", + "direct_io.zig", "gcs.zig", "hf.zig", "http.zig", - "parallel_read.zig", + "http_acceptance_test.zig", + "mock_server.zig", + "range_read.zig", + "request.zig", "s3.zig", ], main = "vfs.zig", diff --git a/vfs/base.zig b/vfs/base.zig index 1e66ca89..fc21c0b7 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -1,9 +1,50 @@ const std = @import("std"); +const builtin = @import("builtin"); const stdx = @import("stdx"); +/// `std.log.scoped`, except that test builds log `err` at `debug`: the test +/// runner counts logged errors as failures, and a test of a failure path +/// checks the error it returns. +pub fn scopedLog(comptime scope: @EnumLiteral()) type { + const scoped = std.log.scoped(scope); + return struct { + pub const debug = scoped.debug; + pub const info = scoped.info; + pub const warn = scoped.warn; + + pub fn err(comptime format: []const u8, args: anytype) void { + if (builtin.is_test) scoped.debug(format, args) else scoped.err(format, args); + } + }; +} + +/// How a backend's files are best read (`VFS.readHints`). No one source +/// request size suits every backend: warm local 8 MiB requests on one B70 +/// reached 27.05 GiB/s against 21.33 at 32 MiB, with a quarter of the pinned +/// memory; real AWS plateaued near 950 MiB/s, and 32 MiB was no faster. +pub const ReadHints = struct { + /// Local files, through a VFS or not. + pub const local: ReadHints = .{ .read_chunk_size = 8 * 1024 * 1024 }; + + /// Smallest source request expected to avoid excessive per-request cost + /// or backend rate limiting. + read_chunk_size: usize = 16 * 1024 * 1024, + /// The backend benefits from opening enough source calls to hide request + /// latency before the first response arrives. + high_latency: bool = false, +}; + +pub const Backend = struct { + io: std.Io, + read_hints: ReadHints = .{}, +}; + pub const VFSBase = struct { - inner: std.Io, + /// Aligned like the i96 durations and timestamps a backend holds (its + /// `request.Governor`), so `@fieldParentPtr("base", ...)` needs no + /// `@alignCast`. + inner: std.Io align(@alignOf(std.Io.Duration)), pub fn init(io: std.Io) VFSBase { return .{ .inner = io }; diff --git a/vfs/direct_io.zig b/vfs/direct_io.zig new file mode 100644 index 00000000..4d758589 --- /dev/null +++ b/vfs/direct_io.zig @@ -0,0 +1,366 @@ +//! Direct I/O on the descriptor of a local file. A VFS handle's `Mode` sets +//! the flag on its descriptor when a reader asks for it under a `Policy`, +//! and from then on issues that descriptor's positional reads here rather +//! than through the inner `std.Io`: the kernel answers an unsupported direct +//! read with `EINVAL` or `EFAULT`, which the threaded `std.Io` treats as a +//! programmer error, whereas here it is a refusal that clears the flag and +//! leaves the read to the VFS, buffered. Everything below `Mode` is Linux +//! only and reached from code gated on `supported`. +//! +//! Neither path wins everywhere, and residency alone cannot predict which +//! does: warm replicated Llama-3.1-8B on eight MI300X took ~1.10 s buffered +//! but ~4.53 s forced direct from a slow storage extent; on four GB300 with +//! four NVMe drives in RAID0, direct took ~0.30 s versus ~0.32 s warm +//! buffered. These are loader times, not disk-only rates. +const std = @import("std"); +const builtin = @import("builtin"); + +const linux = std.os.linux; + +const log = std.log.scoped(.@"zml/vfs"); + +pub const supported = builtin.target.os.tag == .linux and @hasField(std.posix.O, "DIRECT"); + +/// Alignment of a direct read's offset, every buffer and the total length. +/// 4 KiB covered the measured filesystems; this is not a statx(STATX_DIOALIGN) +/// query. A filesystem requiring more alignment may reject the first read, +/// after which the VFS demotes the handle to buffered. +pub const alignment: usize = 4 * 1024; + +/// `Policy.auto` reads a file directly when at most this fraction of its +/// sampled pages is cached. +const cold_fraction: f64 = 0.5; + +/// Whether a local file is read past the page cache. A direct read comes +/// from the disk even when the cache holds the data; it also leaves the cache +/// as it found it, so a file read directly is still cold for the next load +/// (the `auto` probe caches what it samples, elsewhere on every probe: +/// `cachedFraction`). +/// Raw warm-cache reads exceeded disk throughput in the measured setups, +/// but the loader can still favor direct reads: page-cache copies compete +/// with DMA for host memory bandwidth. Conversely, repeated loads from a +/// slow disk can benefit greatly from warming the cache with `off`. The +/// contrasting GB300/MI300X load results are in this file's header. +pub const Policy = enum { + off, + /// Every file the filesystem allows. + on, + /// A file mostly out of the page cache when asked, buffered for a cached + /// one. Residency is sampled: `residency_samples` pages spread over the + /// file, read with `RWF_NOWAIT`, which answers `EAGAIN` for a page that + /// is not cached, needs no permission on the file and costs the same + /// whatever its size. Unknown counts as cached. + auto, +}; + +/// A local handle's read mode: undecided until a reader asks for direct +/// reads (`VFS.useDirectIo`), then buffered or direct for the rest of the +/// handle's life, since the flag belongs to the open file description and a +/// reader that was answered buffered plans exact reads that a direct +/// descriptor would reject. A direct handle goes back to buffered once the +/// kernel rejected a read, or a read went through the inner `Io`. Each +/// method takes the inner `Io` and the handle's file on it. +pub const Mode = struct { + /// Changes only under `mutex`, together with the flag; reads only load it. + state: std.atomic.Value(State) = .init(.undecided), + mutex: std.Io.Mutex = .init, + + const State = enum(u8) { undecided, buffered, direct }; + + /// Whether `file` reads direct, deciding an undecided handle under + /// `policy` (`VFS.useDirectIo`); `off` decides nothing. + pub fn decide(self: *Mode, io: std.Io, file: std.Io.File, policy: Policy) bool { + if (comptime !supported) return false; + if (policy == .off) return false; + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + const stat = file.stat(io) catch return self.settle(io, file, false); + if (!fits(stat)) return self.settle(io, file, false); + if (policy == .auto) { + const name: Name = .{ .io = io, .file = file }; + const cached = sampleResidency(io, file, stat.size) orelse { + log.debug("{f}: page cache residency unknown; reading buffered", .{name}); + return self.settle(io, file, false); + }; + if (cached > cold_fraction) { + log.debug("{f}: {d:.0}% cached; reading buffered", .{ name, cached * 100 }); + return self.settle(io, file, false); + } + log.debug("{f}: {d:.0}% cached", .{ name, cached * 100 }); + } + return self.settle(io, file, true); + } + + /// Decides an undecided handle under the mutex, setting the flag first + /// when `direct`, and returns whether the handle reads direct: a + /// decision already taken (by a racing call, or a demotion) stands. A + /// descriptor the flag cannot go on (`enable`) is read buffered. + fn settle(self: *Mode, io: std.Io, file: std.Io.File, direct: bool) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + switch (self.state.load(.acquire)) { + .direct => return true, + .buffered => return false, + .undecided => {}, + } + if (direct) { + const name: Name = .{ .io = io, .file = file }; + const err = enable(file.handle); + if (err == .SUCCESS) { + self.state.store(.direct, .release); + log.debug("{f}: reading direct", .{name}); + return true; + } + log.debug("{f}: O_DIRECT refused ({t}); reading buffered", .{ name, err }); + } + self.state.store(.buffered, .release); + return false; + } + + /// The bytes read from `offset` into `data` past the page cache, or + /// null for the caller to read them buffered: the handle is not direct, + /// or the kernel refused the read and the handle is buffered from now + /// on (`logRefusal`). A refusal demotes rather than fails: a legitimate + /// short direct read can leave an unaligned continuation, and a + /// filesystem accepting the flag does not prove it accepts these + /// buffers (including PJRT-pinned memory) at read time. + pub fn read(self: *Mode, io: std.Io, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!?usize { + if (comptime !supported) return null; + if (self.state.load(.acquire) != .direct) return null; + switch (try readPositional(file.handle, data, offset)) { + .bytes => |bytes| return bytes, + .refused => |err| { + if (self.demote(io, file)) logRefusal(.{ .io = io, .file = file }, err, data, offset); + return null; + }, + } + } + + /// Takes a direct handle back to buffered before a read the VFS does + /// not issue itself (streaming, or as the source of a file copy): the + /// inner `Io` treats the errno of a rejected direct read as a + /// programmer error. + pub fn leave(self: *Mode, io: std.Io, file: std.Io.File) void { + if (comptime !supported) return; + if (self.state.load(.acquire) != .direct or !self.demote(io, file)) return; + const name: Name = .{ .io = io, .file = file }; + log.warn("{f}: buffered read of a direct file; reading it buffered from now on", .{name}); + } + + /// Takes a direct handle back to buffered: the flag comes off the + /// descriptor before the state says buffered, under the mutex, so the + /// inner `Io` never reads a descriptor with the flag on. True for the + /// call that did it, which then says why. + fn demote(self: *Mode, io: std.Io, file: std.Io.File) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.state.load(.acquire) != .direct) return false; + disable(file.handle); + self.state.store(.buffered, .release); + return true; + } +}; + +/// A file's size, and whether `Mode.decide` would read it direct +/// (`predict`). +pub const Prediction = struct { size: u64, direct: bool }; + +/// `Mode.decide`'s answer for `file` under `policy`, deciding and logging +/// nothing: the same rule, then the flag set on `file` and left set, so +/// `file` must be a read-only descriptor of the caller's own, which the +/// caller closes. Under `auto` the sample caches what it misses, as +/// `decide`'s does: a missed sample queues the read of its page and of the +/// kernel's readahead. A decision that follows and samples those pages +/// once the reads have landed finds the file cached and reads it buffered, +/// which may happen to any file, the likelier the fewer pages it has per +/// sample (`cachedFraction`). Null when `file` cannot be stat-ed. +pub fn predict(io: std.Io, file: std.Io.File, policy: Policy) error{Canceled}!?Prediction { + if (comptime !supported) return null; + const stat = file.stat(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + const direct = fits(stat) and switch (policy) { + .off => false, + .on => true, + .auto => if (sampleResidency(io, file, stat.size)) |cached| cached <= cold_fraction else false, + }; + return .{ .size = stat.size, .direct = direct and enable(file.handle) == .SUCCESS }; +} + +/// A regular file at least one unit long: the only kind read direct +/// (`Mode.decide`, `predict`). +fn fits(stat: std.Io.File.Stat) bool { + return stat.kind == .file and stat.size >= alignment; +} + +/// A local file's path in a log line, resolved only when the line is +/// written. +const Name = struct { + io: std.Io, + file: std.Io.File, + + pub fn format(self: Name, writer: *std.Io.Writer) !void { + var buffer: [std.fs.max_path_bytes]u8 = undefined; + const len = self.file.realPath(self.io, &buffer) catch return writer.writeAll(""); + try writer.writeAll(buffer[0..len]); + } +}; + +/// The kernel rejected a direct read. An aligned one is the filesystem +/// refusing direct I/O it accepted the flag for, a warning; a misaligned +/// one is the reader's, either its plan or a continuation after a short +/// read (which a network filesystem may answer), an error. Either way the +/// file is read buffered from now on. +fn logRefusal(name: Name, err: linux.E, data: []const []u8, offset: u64) void { + var total: usize = 0; + var aligned = offset % alignment == 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + aligned = aligned and @intFromPtr(buffer.ptr) % alignment == 0; + total += buffer.len; + } + aligned = aligned and total % alignment == 0; + if (aligned) { + log.warn("{f}: direct read refused ({t}) at offset={d} total={d}; reading the file buffered from now on", .{ name, err, offset, total }); + } else { + log.err("{f}: misaligned read of a direct file ({t}) at offset={d} first_buffer=0x{x} total={d} alignment={d}; reading the file buffered from now on", .{ + name, + err, + offset, + if (data.len != 0) @intFromPtr(data[0].ptr) else 0, + total, + alignment, + }); + } +} + +const max_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; +const direct_flag: usize = @as(u32, @bitCast(std.posix.O{ .DIRECT = true })); + +/// Sets the flag on a descriptor opened for reading only, since the flag +/// constrains writes too: one opened for writing is refused with `BADF`. A +/// filesystem without direct I/O refuses the flag. +pub fn enable(fd: std.posix.fd_t) linux.E { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => |err| return err, + }; + const status: std.posix.O = @bitCast(@as(u32, @truncate(current))); + if (status.ACCMODE != .RDONLY) return .BADF; + return setStatusFlags(fd, current | direct_flag); +} + +/// Clears the flag, so the inner `Io` may read the descriptor again. +fn disable(fd: std.posix.fd_t) void { + const current = switch (getStatusFlags(fd)) { + .flags => |flags| flags, + .err => return, + }; + _ = setStatusFlags(fd, current & ~direct_flag); +} + +/// `fcntl(F_GETFL)`, retried on `EINTR`. +fn getStatusFlags(fd: std.posix.fd_t) union(enum) { flags: usize, err: linux.E } { + while (true) { + const rc = linux.fcntl(fd, linux.F.GETFL, 0); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .flags = rc }, + .INTR => continue, + else => |err| return .{ .err = err }, + } + } +} + +/// `fcntl(F_SETFL)`, retried on `EINTR`. +fn setStatusFlags(fd: std.posix.fd_t, flags: usize) linux.E { + while (true) { + switch (linux.errno(linux.fcntl(fd, linux.F.SETFL, flags))) { + .INTR => continue, + else => |err| return err, + } + } +} + +/// A fixed count bounds the probe's cost whatever the checkpoint's size and +/// suffices for the coarse half-cached decision. A whole-file `cachestat` +/// walks every cached folio, about 7 ms per cached GiB on a 4 KiB-page +/// kernel, paid on the planner thread, and refuses a file the process can +/// read but not write. +pub const residency_samples = 32; + +/// The fraction of `residency_samples` pages spread over the file that the +/// page cache holds, or null when the filesystem cannot say (no +/// `RWF_NOWAIT`). One sample per window of the file, each `shift` bytes +/// into its window (`shift` modulo the window), unknown residency selecting +/// buffered IO. A sample that misses queues the read of its page, which a +/// direct file never reads back: with the same offsets the next probe would +/// find every sample cached and read a cold file buffered, so callers draw a +/// new `shift` for every probe. A file of at most `residency_samples` pages +/// has every page sampled whatever the shift, so its first probe caches it. +/// The header page a parser has just read is sampled only when `shift` +/// lands in it, not systematically. +fn cachedFraction(fd: std.posix.fd_t, size: u64, shift: u64) ?f64 { + var buffer: [512]u8 = undefined; + const iovec: std.posix.iovec = .{ .base = &buffer, .len = buffer.len }; + var cached: usize = 0; + const window = size / residency_samples; + for (0..residency_samples) |sample| { + const offset = window * sample + shift % @max(window, 1); + const rc = linux.preadv2(fd, @ptrCast(&iovec), 1, @intCast(offset), linux.RWF.NOWAIT); + switch (linux.errno(rc)) { + .SUCCESS => cached += @intFromBool(rc != 0), + .AGAIN => {}, + else => return null, + } + } + return @as(f64, @floatFromInt(cached)) / @as(f64, @floatFromInt(residency_samples)); +} + +/// `cachedFraction` at a fresh shift, as every probe draws one. +fn sampleResidency(io: std.Io, file: std.Io.File, size: u64) ?f64 { + var shift: u64 = undefined; + io.random(std.mem.asBytes(&shift)); + return cachedFraction(file.handle, size, shift); +} + +const ReadResult = union(enum) { + bytes: usize, + /// The kernel rejected the direct read: an unaligned offset, buffer or + /// total, a filesystem that took the flag but not the read, or memory + /// the block layer cannot pin. + refused: linux.E, +}; + +/// Reads `data` at `offset` on a descriptor with the flag set, at most +/// `IOV_MAX` buffers per call (the caller loops). A read that runs past the +/// end of the file is cut there. +fn readPositional(fd: std.posix.fd_t, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!ReadResult { + var iovecs: [max_iovecs]std.posix.iovec = undefined; + var count: usize = 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + if (count == iovecs.len) break; + iovecs[count] = .{ .base = buffer.ptr, .len = buffer.len }; + count += 1; + } + if (count == 0) return .{ .bytes = 0 }; + while (true) { + const rc = linux.preadv(fd, &iovecs, count, @intCast(offset)); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .bytes = rc }, + .INTR => continue, + .INVAL, .FAULT, .OPNOTSUPP => |err| return .{ .refused = err }, + .IO => return error.InputOutput, + .NOMEM => return error.SystemResources, + .AGAIN => return error.WouldBlock, + .ISDIR => return error.IsDir, + .BADF => return error.NotOpenForReading, + .NXIO, .SPIPE => return error.Unseekable, + else => |err| return std.posix.unexpectedErrno(err), + } + } +} diff --git a/vfs/file.zig b/vfs/file.zig deleted file mode 100644 index 1b077c33..00000000 --- a/vfs/file.zig +++ /dev/null @@ -1,430 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs/file"); - -pub const DirectIoError = error{ - UnsupportedPlatform, - MisalignedBuffer, - BufferTooSmall, - UnexpectedFcntlResult, -}; - -fn canUseDirectIO() bool { - if (builtin.target.os.tag == .linux) { - return @hasField(std.posix.O, "DIRECT"); - } - return false; -} - -fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { - while (true) { - const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); - switch (std.posix.errno(result)) { - .SUCCESS => return result, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { - while (true) { - switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { - .SUCCESS => return, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn useDirectIO(file: std.Io.File) DirectIoError!bool { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - return (flags & direct_flag) != 0; - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { - if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - if ((flags & direct_flag) == 0) return; - - try setStatusFlags(file, flags & ~direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -fn switchToDirectIO(file: std.Io.File) DirectIoError!void { - if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try getStatusFlags(file); - const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - try setStatusFlags(file, flags | direct_flag); - } else { - return DirectIoError.UnsupportedPlatform; - } -} - -pub const File = struct { - pub const Config = struct { - direct_io: bool = true, - direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), - }; - - const Handle = struct { - inner_handle: std.Io.File.Handle, - inner_flags: std.Io.File.Flags, - direct_io_handle: ?std.Io.File.Handle, - - pub fn innerFile(self: *Handle) std.Io.File { - return .{ - .handle = self.inner_handle, - .flags = self.inner_flags, - }; - } - - pub fn directIoFile(self: *Handle) std.Io.File { - return .{ - .handle = self.direct_io_handle.?, - .flags = self.inner_flags, - }; - } - }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, - config: Config, - base: VFSBase, - - pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { - return .{ - .allocator = allocator, - .config = config, - .base = .init(inner), - }; - } - - pub fn deinit(self: *File) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - } - - pub fn io(self: *File) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *File) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, self.handles.at(idx) }; - } - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *File, idx: u32) !void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *File, file: std.Io.File) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.handles.at(@intCast(file.handle)); - } - - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { - const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { - else => return std.Io.File.OpenError.Unexpected, - }; - - return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); - } - - fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle == null) { - return handle.innerFile(); - } - - const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); - - if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { - return handle.innerFile(); - } - - var total_size: usize = 0; - for (data) |buf| { - if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { - return handle.innerFile(); - } - total_size += buf.len; - } - - if (!std.mem.isAligned(total_size, alignment_bytes)) { - return handle.innerFile(); - } - - return handle.directIoFile(); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); - const inner_file = self.innerFile(o.file, o.data, position); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_read_streaming = .{ - .file = inner_file, - .data = o.data, - }, - }); - }, - .file_write_streaming => |o| { - const handle = self.getFileHandle(o.file); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_write_streaming = .{ - .file = handle.innerFile(), - .header = o.header, - .data = o.data, - .splat = o.splat, - }, - }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); - } - - fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = null, - }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); - - var direct_io_handle: ?std.Io.File.Handle = null; - if (use_direct_io) { - const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); - defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); - - switchToDirectIO(direct_io_file) catch |err| switch (err) { - DirectIoError.UnsupportedPlatform => { - log.debug("Direct I/O is not supported on this platform", .{}); - }, - else => { - log.err("Failed to switch to Direct I/O mode: {any}", .{err}); - return std.Io.File.OpenError.Unexpected; - }, - }; - - if (useDirectIO(direct_io_file) catch false) { - direct_io_handle = direct_io_file.handle; - } - } - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = direct_io_handle, - }; - - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle = self.getFileHandle(file); - - if (handle.direct_io_handle) |_| { - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); - } - - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); - - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = self.innerFile(file, data, offset); - return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); - } -}; diff --git a/vfs/gcs.zig b/vfs/gcs.zig index acc7f634..6717c59b 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -3,13 +3,15 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/gcs"); -const EndpointUrl = "https://storage.googleapis.com"; const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; +const TokenUrl = "https://oauth2.googleapis.com/token"; /// Check if running on Google Compute Engine / Cloud Run host infrastructure, /// because metadata server probing should only happen on GCP. @@ -51,29 +53,11 @@ const Credentials = union(enum) { }; const ServiceAccount = struct { - pub const JwtAssertion = struct { - iss: []const u8, - scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", - aud: []const u8 = "https://oauth2.googleapis.com/token", - exp: i64, - iat: i64, - sub: ?[]const u8 = null, - }; - type: []const u8, client_email: []const u8, private_key: []const u8, token_uri: []const u8, quota_project_id: ?[]const u8 = null, - - pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { - const now: std.Io.Timestamp = .now(io_, .real); - return .{ - .iss = self.client_email, - .iat = now.toSeconds(), - .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), - }; - } }; authorized_user: AuthorizedUser, @@ -122,159 +106,36 @@ const Credentials = union(enum) { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const GCS = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *GCS, - uri: std.Uri, - path: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request( - .GET, - job.batch.uri, - .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }, - ) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; + const authorization_header_size = "Bearer ".len + OAuthToken.Size; const Token = struct { - header: []u8, + /// Guards the token. A refresh is a governed request that may wait + /// out a hold or retry for minutes, so it runs under this lock, not + /// under `GCS.mutex`, which every handle lookup takes. + mutex: std.Io.Mutex = .init, + header_storage: []u8, + header_len: usize = 0, expires_at: std.Io.Timestamp, + /// Refreshes that failed other than by a cancel. A reader that + /// queued behind one fails with it instead of running its own. + failures: std.atomic.Value(u64) = .init(0), fn expired(self: *const Token, io_: std.Io) bool { const now: std.Io.Timestamp = .now(io_, .real); return now.toSeconds() >= self.expires_at.toSeconds(); } + + fn header(self: *const Token) []const u8 { + return self.header_storage[0..self.header_len]; + } }; pub const Config = struct { credentials: ?Credentials = null, endpoint_uri: std.Uri, - region: []const u8 = "auto", + /// Where tokens come from; a test points them at a local server. + metadata_uri: std.Uri = std.Uri.parse(MetadataUrl) catch unreachable, + token_uri: std.Uri = std.Uri.parse(TokenUrl) catch unreachable, }; const Handle = struct { @@ -302,49 +163,33 @@ pub const GCS = struct { allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, token: Token, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub const InitOpts = struct { + pub const Options = struct { credentials: ?union(enum) { json: *std.Io.Reader, metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", - region: []const u8 = "auto", - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + retry: request.Retry = .{}, }; pub const InitError = error{ InvalidCredentialJson, Unexpected, - } || std.mem.Allocator.Error || std.Io.ConcurrentError; + } || std.mem.Allocator.Error; - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) InitError!GCS { var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); - - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - const config: Config = .{ - .credentials = if (opts.credentials) |creds| switch (creds) { + .credentials = if (options.credentials) |creds| switch (creds) { .json => |reader| blk: { var json_reader: std.json.Reader = .init(allocator, reader); defer json_reader.deinit(); @@ -358,12 +203,11 @@ pub const GCS = struct { }, .metadata_server => .{ .metadata_server = {} }, } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, - .region = try arena.allocator().dupe(u8, opts.region), + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, options.endpoint_url)) catch return InitError.Unexpected, }; const token: Token = .{ - .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .header_storage = try arena.allocator().alloc(u8, authorization_header_size), .expires_at = .zero, }; @@ -371,10 +215,10 @@ pub const GCS = struct { .allocator = allocator, .arena = arena, .base = .init(inner), - .client = http_client, .config = config, .token = token, - .read_pool = read_pool, + // GCS answers rate limiting with `503` as well as `429`. + .governor = .init(inner, http_client, "gcs", .throttle, options.retry), }; } @@ -419,129 +263,120 @@ pub const GCS = struct { return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; } - fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = MetadataUrl }, - .method = .GET, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - }, - .extra_headers = &.{ - .{ .name = "Metadata-Flavor", .value = "Google" }, - }, - .response_writer = &response_writer, - }); - - if (result.status != .ok) { - return null; - } - - return response_writer.buffered(); + /// The metadata server's token, through the governed loop: a throttled + /// token endpoint holds the backend like any other request. + fn refreshMetadataServerToken(self: *GCS, buffer: []u8) ![]const u8 { + var token: TokenRequest = .{ + .uri = self.config.metadata_uri, + .target = "the metadata server", + .buffer = buffer, + .extra_headers = &.{.{ .name = "Metadata-Flavor", .value = "Google" }}, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } - fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = "https://oauth2.googleapis.com/token" }, + fn refreshAuthorizedUserToken(self: *GCS, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + // Its own buffer: a retried attempt sends it again after the + // response was written into `buffer`. + var payload_buffer: [2 * 1024]u8 = undefined; + const payload = try std.fmt.bufPrint(&payload_buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }); + var token: TokenRequest = .{ + .uri = self.config.token_uri, + .target = "the ADC token endpoint", + .buffer = buffer, .method = .POST, - .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - }), - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .content_type = .{ .override = "application/x-www-form-urlencoded" }, - }, - .response_writer = &response_writer, - }); - if (result.status != .ok) { - log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - return error.RequestFailed; - } - return response_writer.buffered(); + .payload = payload, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }; + return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); } + /// One OAuth token exchange, whose body it copies into `buffer`. + const TokenRequest = struct { + uri: std.Uri, + target: []const u8, + buffer: []u8, + method: std.http.Method = .GET, + payload: ?[]u8 = null, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + + fn spec(self: *const TokenRequest) request.RequestSpec { + return .{ .target = self.target, .key = request.authorityOf(self.uri) }; + } + + fn attempt(self: *TokenRequest, current: request.Attempt) anyerror!request.Outcome([]const u8) { + return request.exchange([]const u8, current, self.uri, .{ + .method = self.method, + .content_type = self.content_type, + .extra_headers = self.extra_headers, + .payload = self.payload, + .redirects = .follow, + // A refused exchange's body says why (`invalid_grant`, ...). + .log_refusal_body = true, + }, self, TokenRequest.consume); + } + + fn consume(self: *TokenRequest, res: *std.http.Client.Response) anyerror![]const u8 { + var writer: std.Io.Writer = .fixed(self.buffer); + _ = try res.reader(&.{}).streamRemaining(&writer); + return writer.buffered(); + } + }; + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { _ = io_; _ = client; _ = service_account; _ = buffer; return error.Unimplemented; - - // const assertion = service_account.makeAssertion(io_); - - // var tmp_buffer: [8 * 1024]u8 = undefined; - - // var buffer_writer: std.Io.Writer = .fixed(buffer); - // var signing_input_buffer: [2 * 1024]u8 = undefined; - // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); - - // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; - // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); - // std.base64.url_safe_no_pad.Encoder.encodeWriter( - // signing_input_writer, - // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ - // std.json.fmt(assertion, .{}), - // }), - // ); - // const signing_input = signing_input_writer.buffered(); - // _ = signing_input; - - // const result = try client.fetch(.{ - // .location = .{ .url = "https://oauth2.googleapis.com/token" }, - // .method = .POST, - // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - // }), - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .content_type = .{ .override = "application/x-www-form-urlencoded" }, - // }, - // .response_writer = &response_writer, - // }); - // if (result.status != .ok) { - // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - // return error.RequestFailed; - // } } fn refreshToken(self: *GCS) !void { var buffer: [4 * 1024]u8 = undefined; const payload = switch (self.config.credentials.?) { - .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), - .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), - .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + .authorized_user => |authorized_user| try self.refreshAuthorizedUserToken(authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.governor.client, service_account, &buffer), + .metadata_server => try self.refreshMetadataServerToken(&buffer), }; const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ .allocate = .alloc_if_needed, .ignore_unknown_fields = true, }); defer oauth_token.deinit(); - self.token = .{ - .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), - .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), - }; + const header = try std.fmt.bufPrint(self.token.header_storage, "Bearer {s}", .{oauth_token.value.access_token}); + self.token.header_len = header.len; + self.token.expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))); } - fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + fn getOrRefreshToken(self: *GCS, authorization_buffer: []u8) !std.http.Client.Request.Headers.Value { if (self.config.credentials == null) { return .omit; } + const failures = self.token.failures.load(.monotonic); + try self.token.mutex.lock(self.base.inner); + defer self.token.mutex.unlock(self.base.inner); if (self.token.expired(self.base.inner)) { - try self.refreshToken(); + // The refresh this reader waited behind failed, and its reader + // logged why: repeating it, with a retry budget of its own, would + // only keep every other queued reader waiting. + if (self.token.failures.load(.monotonic) != failures) return error.RequestFailed; + self.refreshToken() catch |err| { + if (err != error.Canceled) _ = self.token.failures.fetchAdd(1, .monotonic); + return err; + }; } - return .{ .override = self.token.header }; + // A copy: a refresh by another reader rewrites the token's storage. + return .{ .override = try std.fmt.bufPrint(authorization_buffer, "{s}", .{self.token.header()}) }; } pub fn deinit(self: *GCS) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -591,6 +426,13 @@ pub const GCS = struct { }; } + pub fn backend(self: *GCS) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *GCS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -640,9 +482,9 @@ pub const GCS = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -688,11 +530,9 @@ pub const GCS = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -707,18 +547,19 @@ pub const GCS = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -744,7 +585,16 @@ pub const GCS = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -829,10 +679,7 @@ pub const GCS = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -842,7 +689,7 @@ pub const GCS = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -864,25 +711,6 @@ pub const GCS = struct { return path.len; } - fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { - const now_ts = std.Io.Clock.now(.real, self.base.inner); - - const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); - const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; - const year_day = epoch_secs.getEpochDay().calculateYearDay(); - const month_day = year_day.calculateMonthDay(); - const day_seconds = epoch_secs.getDaySeconds(); - - return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ - year_day.year, - @backingInt(month_day.month), - month_day.day_index + 1, - day_seconds.getHoursIntoDay(), - day_seconds.getMinutesIntoHour(), - day_seconds.getSecondsIntoMinute(), - }); - } - fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { const path = std.mem.trim(u8, path_, "/"); @@ -893,43 +721,16 @@ pub const GCS = struct { } } - fn gcsUri(self: *GCS, path: []const u8) std.Uri { + /// The object's URI. A VFS path is `bucket/key`; the request target + /// needs `/bucket/key`, formatted into `buffer`, which the URI borrows. + fn gcsUri(self: *GCS, path: []const u8, buffer: []u8) !std.Uri { var uri = self.config.endpoint_uri; - uri.path = .{ .raw = path }; + const absolute = if (std.mem.startsWith(u8, path, "/")) path else std.fmt.bufPrint(buffer, "/{s}", .{path}) catch return error.NameTooLong; + uri.path = .{ .raw = absolute }; uri.query = .{ .percent_encoded = "alt=media" }; return uri; } - // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { - // authorization: ?[]const u8, - // extra_headers: [3]?std.http.Header, - // extra_len: usize, - // } { - // if (self.config.oauth_access_token == null) { - // if (self.config.googleCredentialsJsonStr) |json| { - // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); - // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); - // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); - // self.config.oauth_access_token = oauth.access_token; - // self.config.quota_project_id = oauth.quota_project_id; - // } - // } - - // const token = self.config.oauth_access_token orelse return error.MissingCredentials; - // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); - // var extra_headers: [3]?std.http.Header = .{ null, null, null }; - // var extra_len: usize = 0; - // if (self.config.quota_project_id) |project| { - // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; - // extra_len = 1; - // } - // return .{ - // .authorization = authorization, - // .extra_headers = extra_headers, - // .extra_len = extra_len, - // }; - // } - fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -951,35 +752,35 @@ pub const GCS = struct { uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; uri.query = .{ .percent_encoded = query_writer.buffered() }; - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = if (self.quotaProjectId()) |project| - &.{.{ .name = "x-goog-user-project", .value = project }} - else - &.{}, - }); - defer req.deinit(); + var listing: Listing = .{ .gcs = self, .uri = uri }; + return request.perform([]u8, &self.governor, .{ .target = bucket, .key = request.authorityOf(uri) }, &listing, Listing.attempt); + } - try req.sendBodiless(); + /// The bucket listing. The bearer token is copied per attempt, so one + /// that expired during a hold is refreshed before the next try. + const Listing = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .extra_headers = if (self.gcs.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.gcs.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { const bucket, const key_prefix = self.pathComponents(prefix); @@ -1030,76 +831,65 @@ pub const GCS = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - const path = try self.resolvePath(dir, sub_path, &path_buffer); - const uri = self.gcsUri(path); - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = &.{}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - const size = switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - .unauthorized, .forbidden => return error.PermissionDenied, - else => blk: { - log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; - return size; + fn fetchSize(self: *GCS, path: []const u8) !u64 { + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + const uri = try self.gcsUri(path, &uri_path_buffer); + var head: SizeRequest = .{ .gcs = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = path, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one HEAD. An absent or denied object fails the + /// request (`ExchangeOptions.open`). + const SizeRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, - const uri = self.gcsUri(handle.uri); - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .path = handle.uri, - .authorization = try self.getOrRefreshToken(), - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.gcs.getOrRefreshToken(&self.authorization), + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; + var bearer: BearerRequest = .{ .gcs = self, .uri = try self.gcsUri(handle.uri, &uri_path_buffer) }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = handle.uri, .key = request.authorityOf(bearer.uri) }, + }, &bearer, BearerRequest.prepare, data, offset, handle.size); } + + /// The bearer token is copied per attempt, so one that expired during a + /// retry delay is refreshed before the next GET. + const BearerRequest = struct { + gcs: *GCS, + uri: std.Uri, + authorization: [authorization_header_size]u8 = undefined, + + fn prepare(self: *BearerRequest, _: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ .uri = self.uri, .authorization = try self.gcs.getOrRefreshToken(&self.authorization) }; + } + }; }; +test "GCS object URIs are rooted at the endpoint" { + var backend: GCS = undefined; + backend.config = .{ .endpoint_uri = try std.Uri.parse("https://storage.googleapis.com") }; + + var path_buffer: [64]u8 = undefined; + const uri = try backend.gcsUri("bucket/dir/object.bin", &path_buffer); + var uri_buffer: [128]u8 = undefined; + const formatted = try std.fmt.bufPrint(&uri_buffer, "{f}", .{uri}); + try std.testing.expectEqualStrings("https://storage.googleapis.com/bucket/dir/object.bin?alt=media", formatted); +} + test "GCS parses XML bucket listing objects and common prefixes" { const xml = "" ++ @@ -1136,3 +926,272 @@ test "GCS splits bucket and object prefix" { try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); try std.testing.expectEqualStrings("LC08/01", prefix); } + +/// A token endpoint at `/token` and one object at `/bucket/object`, for the +/// token tests. The nth token request gets `answers[n]`, the last repeating; +/// a 200 grants `tok-`, k counting the grants. +const TokenServer = struct { + tcp: std.Io.net.Server, + group: std.Io.Group = .init, + object: []const u8, + answers: []const std.http.Status, + /// How long each token answer takes, so that readers queue behind a + /// refresh. + delay: std.Io.Duration = .fromMilliseconds(100), + /// The body every token request must POST form-encoded, if any. + form: ?[]const u8 = null, + token_requests: std.atomic.Value(usize) = .init(0), + granted: std.atomic.Value(usize) = .init(0), + object_requests: std.atomic.Value(usize) = .init(0), + /// Object requests without the last token granted. + stale_requests: std.atomic.Value(usize) = .init(0), + /// Token requests that did not POST `form`. + bad_forms: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + + fn init(io: std.Io, object: []const u8, answers: []const std.http.Status) !TokenServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ .tcp = try address.listen(io, .{ .reuse_address = true }), .object = object, .answers = answers }; + } + + fn deinit(self: *TokenServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + fn run(self: *TokenServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) std.Io.Cancelable!void { + defer stream.close(io); + self.answer(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn answer(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var req = try server.receiveHead(); + + if (std.mem.eql(u8, req.head.target, "/token")) { + const n = self.token_requests.fetchAdd(1, .acq_rel); + if (self.form) |form| { + const content_type = req.head.content_type orelse ""; + const form_encoded = req.head.method == .POST and std.mem.eql(u8, content_type, "application/x-www-form-urlencoded"); + var body_buffer: [512]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + _ = try req.readerExpectNone(&.{}).streamRemaining(&body); + if (!form_encoded or !std.mem.eql(u8, body.buffered(), form)) _ = self.bad_forms.fetchAdd(1, .acq_rel); + } + try io.sleep(self.delay, .awake); + const status = self.answers[@min(n, self.answers.len - 1)]; + if (status != .ok) return req.respond("{\"error\":\"invalid_grant\"}", .{ .status = status, .keep_alive = false }); + var token_buffer: [64]u8 = undefined; + const token = try std.fmt.bufPrint(&token_buffer, "{{\"access_token\":\"tok-{d}\",\"expires_in\":3600}}", .{self.granted.fetchAdd(1, .acq_rel) + 1}); + return req.respond(token, .{ .keep_alive = false }); + } + + if (!std.mem.eql(u8, req.head.target, "/bucket/object?alt=media")) return error.UnexpectedTarget; + _ = self.object_requests.fetchAdd(1, .acq_rel); + var authorization: []const u8 = ""; + var range: ?[]const u8 = null; + var headers = req.iterateHeaders(); + while (headers.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "authorization")) authorization = header.value; + if (std.ascii.eqlIgnoreCase(header.name, "range")) range = header.value; + } + var expected_buffer: [32]u8 = undefined; + const expected = try std.fmt.bufPrint(&expected_buffer, "Bearer tok-{d}", .{self.granted.load(.acquire)}); + if (!std.mem.eql(u8, authorization, expected)) _ = self.stale_requests.fetchAdd(1, .acq_rel); + if (req.head.method == .HEAD) return req.respond(self.object, .{ .keep_alive = false }); + + const spec = (range orelse return error.MissingRange)["bytes=".len..]; + const dash = std.mem.indexOfScalar(u8, spec, '-') orelse return error.InvalidRange; + const first = try std.fmt.parseInt(usize, spec[0..dash], 10); + const last = try std.fmt.parseInt(usize, spec[dash + 1 ..], 10); + var content_range_buffer: [64]u8 = undefined; + return req.respond(self.object[first .. last + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{.{ + .name = "Content-Range", + .value = try std.fmt.bufPrint(&content_range_buffer, "bytes {d}-{d}/{d}", .{ first, last, self.object.len }), + }}, + }); + } +}; + +/// A GCS whose tokens and object come from a `TokenServer`. It must not +/// move once set up. +const TokenFixture = struct { + object: [4096]u8 = undefined, + server: TokenServer = undefined, + client: std.http.Client = undefined, + gcs: GCS = undefined, + endpoint_buffer: [32]u8 = undefined, + token_url_buffer: [48]u8 = undefined, + + fn setUp( + self: *TokenFixture, + credentials: @FieldType(GCS.Options, "credentials"), + answers: []const std.http.Status, + max_retries: usize, + ) !void { + const io = std.testing.io; + for (&self.object, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% 3); + self.server = try .init(io, &self.object, answers); + errdefer self.server.deinit(io); + try self.server.group.concurrent(io, TokenServer.run, .{ &self.server, io }); + self.client = .{ .allocator = std.testing.allocator, .io = io }; + errdefer self.client.deinit(); + const port = self.server.tcp.socket.address.getPort(); + const token_uri: std.Uri = try .parse(try std.fmt.bufPrint(&self.token_url_buffer, "http://127.0.0.1:{d}/token", .{port})); + self.gcs = try .init(std.testing.allocator, io, &self.client, .{ + .credentials = credentials, + .endpoint_url = try std.fmt.bufPrint(&self.endpoint_buffer, "http://127.0.0.1:{d}", .{port}), + .retry = .{ + .max_retries = max_retries, + .initial_delay = .fromMilliseconds(5), + .max_delay = .fromMilliseconds(5), + .max_hold = .fromMilliseconds(20), + }, + }); + self.gcs.config.metadata_uri = token_uri; + self.gcs.config.token_uri = token_uri; + } + + fn tearDown(self: *TokenFixture) void { + self.gcs.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } + + /// Stops the server and returns the first error it met. + fn stop(self: *TokenFixture) !void { + self.server.group.cancel(std.testing.io); + if (self.server.first_error.get()) |err| return err; + } + + /// `readers` concurrent reads of 256 bytes each, started together; + /// returns how many failed. Every read that succeeds must match. + fn readConcurrently(self: *TokenFixture, file: std.Io.File, comptime readers: usize) !usize { + const io = std.testing.io; + const Reader = struct { + fn run(gcs_io: std.Io, file_: std.Io.File, output: []u8, offset: u64, go: *std.Io.Event, result: *anyerror!usize) void { + go.wait(gcs_io) catch |err| { + result.* = err; + return; + }; + result.* = file_.readPositional(gcs_io, &.{output}, offset); + } + }; + var outputs: [readers][256]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var go: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, i| { + try group.concurrent(io, Reader.run, .{ self.gcs.io(), file, output, i * output.len, &go, result }); + } + go.set(io); + try group.await(io); + + var failed: usize = 0; + for (&outputs, results, 0..) |*output, result, i| { + const n = result catch { + failed += 1; + continue; + }; + try std.testing.expectEqual(output.len, n); + try std.testing.expectEqualSlices(u8, self.object[i * output.len ..][0..output.len], output); + } + return failed; + } +}; + +test "GCS refreshes an expired token once for concurrent readers, and each sends the new one" { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{.ok}, 0); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + try std.testing.expectEqual(1, server.token_requests.load(.acquire)); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(2, server.token_requests.load(.acquire)); + try std.testing.expectEqual(17, server.object_requests.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} + +test "GCS fails the readers queued behind a failed token refresh without repeating it" { + const cases = [_]struct { status: std.http.Status, max_retries: usize, token_requests: usize }{ + // The open's request, then one refused refresh. + .{ .status = .unauthorized, .max_retries = 0, .token_requests = 2 }, + // The open's request, then one refresh and its 2 retries. + .{ .status = .internal_server_error, .max_retries = 2, .token_requests = 4 }, + }; + for (cases) |case| { + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .metadata_server = {} }, &.{ .ok, case.status }, case.max_retries); + defer fixture.tearDown(); + const server = &fixture.server; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + fixture.gcs.token.expires_at = .zero; + try std.testing.expectEqual(16, try fixture.readConcurrently(file, 16)); + try fixture.stop(); + try std.testing.expectEqual(case.token_requests, server.token_requests.load(.acquire)); + try std.testing.expectEqual(1, server.object_requests.load(.acquire)); + } +} + +test "GCS resends the ADC refresh form on a retry, and a throttled token endpoint holds the backend" { + var json: std.Io.Reader = .fixed( + \\{"type":"authorized_user","client_id":"cid.apps.googleusercontent.com","client_secret":"GOCSPX-sec","refresh_token":"1//0abc-def_ghi","account":"","universe_domain":"googleapis.com"} + ); + var fixture: TokenFixture = .{}; + try fixture.setUp(.{ .json = &json }, &.{ .too_many_requests, .internal_server_error, .ok }, 2); + defer fixture.tearDown(); + const server = &fixture.server; + server.delay = .zero; + server.form = "grant_type=refresh_token&client_id=cid.apps.googleusercontent.com&client_secret=GOCSPX-sec&refresh_token=1//0abc-def_ghi"; + const gcs_io = fixture.gcs.io(); + const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); + defer file.close(gcs_io); + + try std.testing.expectEqual(0, try fixture.readConcurrently(file, 4)); + try fixture.stop(); + const stats = &fixture.gcs.governor.stats; + try std.testing.expectEqual(1, stats.holds.load(.acquire)); + try std.testing.expectEqual(1, stats.retries.load(.acquire)); + try std.testing.expectEqual(3, server.token_requests.load(.acquire)); + try std.testing.expectEqual(0, server.bad_forms.load(.acquire)); + try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); +} diff --git a/vfs/hf.zig b/vfs/hf.zig index 8770fe18..2989d61f 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -2,160 +2,13 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/hf"); -const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - uri: std.Uri, - url: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - pub const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout => .retry(), - .too_many_requests => b: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { - var parts = std.mem.splitScalar(u8, header.value, ';'); - while (parts.next()) |part| { - const p = std.mem.trim(u8, part, " \t"); - if (std.mem.startsWith(u8, p, "t=")) { - const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; - break :b .{ .retry_after = .fromSeconds(seconds) }; - } - } - } - } - break :b .retry(); - }, - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; -}; - pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; @@ -200,16 +53,10 @@ const RepoKey = struct { }; pub const HF = struct { - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - // Chunk size needs to be big enough to avoid hitting the rate limits of HF. - .chunk_size = 32 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, + pub const Options = struct { + /// Sent as a bearer token to the Hub itself, never to its CDN. + token: ?[]const u8 = null, + retry: request.Retry = .{}, }; pub const Repo = struct { @@ -243,13 +90,17 @@ pub const HF = struct { type: Type, uri: []const u8, + download_url: ?[]const u8, pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, download_url: ?[]const u8, size: u64) !Handle { + const owned_uri = try allocator.dupe(u8, uri); + errdefer allocator.free(owned_uri); return .{ .type = handle_type, - .uri = try allocator.dupe(u8, uri), + .uri = owned_uri, + .download_url = if (download_url) |url| try allocator.dupe(u8, url) else null, .pos = 0, .size = size, }; @@ -257,39 +108,37 @@ pub const HF = struct { pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { allocator.free(self.uri); + if (self.download_url) |url| allocator.free(url); } }; allocator: std.mem.Allocator, + /// Guards the handle table and the handles' download URLs. mutex: std.Io.Mutex = .init, - client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, + /// Guards `trees`. A tree fetch is a governed request that may wait out + /// a hold or retry for minutes, so it runs under this lock, not under + /// `mutex`, which every read takes. + trees_mutex: std.Io.Mutex = .init, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HF { var self: HF = .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .authorization = if (hf_token) |token| blk: { + .authorization = if (options.token) |token| blk: { break :blk .{ .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), }; } else blk: { break :blk .default; }, - .read_pool = read_pool, + .governor = .init(inner, http_client, "hf", .server_failure, options.retry), }; errdefer switch (self.authorization) { .default, .omit => {}, @@ -321,13 +170,11 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token, .{}); + return init(allocator, inner, http_client, .{ .token = hf_token }); } pub fn deinit(self: *HF) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -383,6 +230,16 @@ pub const HF = struct { }; } + pub fn backend(self: *HF) Backend { + return .{ + .io = self.io(), + .read_hints = .{ + .read_chunk_size = 32 * 1024 * 1024, + .high_latency = true, + }, + }; + } + fn openHandle(self: *HF) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -429,39 +286,36 @@ pub const HF = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { + /// The repository's listing, fetched once. The lock is held across the + /// fetch, so every open, stat or listing of this backend waits for it, + /// cached or not. The map is read and written under `trees_mutex`, since + /// an insert may move its entries; the nodes themselves never move once + /// fetched, so the returned slice stays valid without the lock. + fn getOrFetchTree(self: *HF, repo: Repo) ![]const TreeNode { const key = repo.toKey(); var cache_key_buffer: [512]u8 = undefined; - const cache_key_lookup = try key.format(&cache_key_buffer); - if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; + try self.trees_mutex.lock(self.base.inner); + defer self.trees_mutex.unlock(self.base.inner); + if (self.trees.get(cache_key_lookup)) |tree| return tree.items; + + // Room first: once fetched, the listing cannot fail to be kept. + try self.trees.ensureUnusedCapacity(self.allocator, 1); const cache_key = try key.allocFormat(self.allocator); errdefer self.allocator.free(cache_key); - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - const gop = try self.trees.getOrPut(self.allocator, cache_key); - errdefer _ = self.trees.remove(cache_key); - - if (gop.found_existing) { - self.allocator.free(cache_key); - return gop.value_ptr; - } - - const tree_root = try self.fetchTreeFromAPI(repo); - gop.value_ptr.* = tree_root; - - return gop.value_ptr; + const tree = try self.fetchTreeFromAPI(repo); + self.trees.putAssumeCapacityNoClobber(cache_key, tree); + return tree.items; } + /// The repository listing, through the governed loop like every other + /// request. It runs under `trees_mutex` from `getOrFetchTree`, never + /// under `mutex`, so reads of open files go on while it waits. fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { var url_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ .repo = repo.repo, .model = repo.model, @@ -471,40 +325,64 @@ pub const HF = struct { const uri = try std.Uri.parse(url); log.info("Fetching from HF: {s}", .{url}); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, - }); - defer req.deinit(); + var fetch: TreeFetch = .{ .hf = self, .uri = uri }; + return request.perform(std.ArrayList(TreeNode), &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &fetch, TreeFetch.attempt); + } - try req.sendBodiless(); + const TreeFetch = struct { + hf: *HF, + uri: std.Uri, - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to fetch tree: status={}", .{res.head.status}); - return error.RequestFailed; + fn attempt(self: *TreeFetch, current: request.Attempt) anyerror!request.Outcome(std.ArrayList(TreeNode)) { + return request.exchange(std.ArrayList(TreeNode), current, self.uri, .{ + .authorization = self.hf.authorization, + .open = true, + .redirects = .follow, + }, self, TreeFetch.consume); } - const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); - defer self.allocator.free(body); + fn consume(self: *TreeFetch, res: *std.http.Client.Response) anyerror!std.ArrayList(TreeNode) { + const allocator = self.hf.allocator; + const body = try res.reader(&.{}).readAlloc(allocator, res.head.content_length orelse return error.MissingContentLength); + defer allocator.free(body); - const parsed = try std.json.parseFromSlice( - []API.Tree, - self.allocator, - body, - .{ .ignore_unknown_fields = true }, - ); - defer parsed.deinit(); + const parsed = try std.json.parseFromSlice( + []API.Tree, + allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .empty; - for (parsed.value) |item| { - try insertTreeNode(self.allocator, &tree_root, item); + var tree_root: std.ArrayList(TreeNode) = .empty; + errdefer { + for (tree_root.items) |*node| node.deinit(allocator); + tree_root.deinit(allocator); + } + for (parsed.value) |item| { + try insertTreeNode(allocator, &tree_root, item); + } + return tree_root; } + }; - return tree_root; + /// The URL the file's bytes are read from, copied into `out_buffer`: + /// where the Hub's redirect chain ends, a signed CDN URL for an LFS file. + fn resolveDownloadUrl(self: *HF, repo: Repo, out_buffer: []u8) ![]const u8 { + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + // Logs name the file: past the first hop the URL is a signed one, + // which is a credential. + var target_buffer: [1024]u8 = undefined; + const target = std.fmt.bufPrint(&target_buffer, "{s}/{s}@{s}/{s}", .{ repo.repo, repo.model, repo.rev, repo.path }) catch repo.path; + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, target, url, 8, self, hubAuthorization); + return std.fmt.bufPrint(out_buffer, "{s}", .{end.url}); + } + + /// Credentials only travel to the Hub itself, never to its CDN. + fn hubAuthorization(self: *HF, url: []const u8) std.http.Client.Request.Headers.Value { + return if (std.mem.startsWith(u8, url, "https://huggingface.co/")) self.authorization else .omit; } fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { @@ -570,16 +448,16 @@ pub const HF = struct { return null; } - fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { + fn getSizeFromTree(tree: []const TreeNode, path: []const u8) ?u64 { if (path.len == 0) { var total: u64 = 0; - for (tree.items) |*node| { + for (tree) |*node| { if (node.kind == .file) total += node.size; } return total; } - const node = findNode(tree.items, path) orelse return null; + const node = findNode(tree, path) orelse return null; return node.size; } @@ -640,9 +518,9 @@ pub const HF = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -665,11 +543,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, null, size) catch return std.Io.Dir.OpenError.Unexpected; return .{ .handle = @intCast(idx) }; } @@ -698,16 +576,16 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; - const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; - + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // The repository itself is a directory. + if (repo.path.len == 0) return std.Io.Dir.StatFileError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; return .{ .inode = @intCast(0), .nlink = 0, - .size = size, + .size = node.size, .permissions = .fromMode(0o444), .kind = .file, .atime = null, @@ -724,11 +602,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); if (repo.path.len == 0) return; - const node = findNode(tree.items, repo.path); + const node = findNode(tree, repo.path); if (node == null) return std.Io.Dir.AccessError.FileNotFound; } @@ -739,11 +617,17 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; - const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + // A directory, the repository itself included, has no download URL. + if (repo.path.len == 0) return std.Io.File.OpenError.IsDir; + const node = findNode(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + if (node.kind == .directory) return std.Io.File.OpenError.IsDir; + var download_url_buffer: [16 * 1024]u8 = undefined; + // Logged by the file's path: its signed URL would be a credential. + const download_url = self.resolveDownloadUrl(repo, &download_url_buffer) catch |err| return self.governor.openFailure(full_path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, download_url, node.size) catch return std.Io.File.OpenError.Unexpected; return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } @@ -765,9 +649,12 @@ pub const HF = struct { const handle = self.getDirHandle(reader.dir); const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; + const tree = self.getOrFetchTree(repo) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return std.Io.Dir.Reader.Error.Unexpected, + }; - const children = findDirChildren(tree.items, repo.path) orelse &.{}; + const children = findDirChildren(tree, repo.path) orelse &.{}; self.dir_read_states.put(self.allocator, reader, .{ .children = children, @@ -845,20 +732,18 @@ pub const HF = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -877,44 +762,125 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var read_size: usize = 0; - for (data) |buf| read_size += buf.len; - read_size = @intCast(@min(handle.size - offset, read_size)); - if (read_size == 0) return 0; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - const repo: Repo = try .parse(handle.uri); - var url_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - const uri: std.Uri = try .parse(url); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .uri = uri, - .url = url, - .authorization = self.authorization, + // Another read of the file may replace the handle's URL + // (`DataRequest.reresolve`): each read sends its own copy. + const url = url: { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + break :url try self.allocator.dupe(u8, handle.download_url orelse return error.InvalidHandle); }; + defer self.allocator.free(url); + var download: DataRequest = .{ .hf = self, .handle = handle, .uri = try .parse(url), .url = url }; + defer if (download.re_resolved) |owned| self.allocator.free(owned); + return range_read.performRangeRead(&self.governor, .{ + .request = .{ + // The file, not its signed URL: that is a credential, and + // a re-resolve replaces it. + .target = handle.uri, + .key = request.authorityOf(download.uri), + // A signed CDN URL can expire while the backend is held; + // the CDN then answers 401 or 403, which is not retryable in + // general but is here, once, after re-resolving it. + .retry_once = &.{ .unauthorized, .forbidden }, + }, + }, &download, DataRequest.prepare, data, offset, handle.size); + } - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, + const DataRequest = struct { + hf: *HF, + handle: *Handle, + /// Parsed from `url`, whose bytes it slices. + uri: std.Uri, + /// The read's own copy of the URL it sends. + url: []const u8, + /// The URL `reresolve` gave this read, owned by the read; at most + /// one per read. + re_resolved: ?[]const u8 = null, + + fn prepare(self: *DataRequest, attempt: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { + if (attempt.previous_status) |status| switch (status) { + .unauthorized, .forbidden => { + if (self.re_resolved != null) return error.PermissionDenied; + try self.reresolve(); + }, + else => {}, + }; + return .{ + .uri = self.uri, + .authorization = self.hf.hubAuthorization(self.url), }; } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; - - return read_size; - } + /// Resolves the download URL again through the governed HEAD chain + /// and stores a copy on the handle, so later reads of the same file + /// use the fresh one too. Reads copy the handle's URL under + /// `hf.mutex`, so none still uses the one this replaces. + fn reresolve(self: *DataRequest) !void { + const allocator = self.hf.allocator; + const repo = try Repo.parse(self.handle.uri); + var url_buffer: [16 * 1024]u8 = undefined; + const resolved = try self.hf.resolveDownloadUrl(repo, &url_buffer); + // Everything that can fail comes before anything is handed over. + const owned = try allocator.dupe(u8, resolved); + errdefer allocator.free(owned); + const uri: std.Uri = try .parse(owned); + const for_handle = try allocator.dupe(u8, resolved); + { + self.hf.mutex.lockUncancelable(self.hf.base.inner); + defer self.hf.mutex.unlock(self.hf.base.inner); + if (self.handle.download_url) |previous| allocator.free(previous); + self.handle.download_url = for_handle; + } + self.re_resolved = owned; + self.url = owned; + self.uri = uri; + log.warn("hf: re-resolved the download URL of {s}", .{self.handle.uri}); + } + }; }; + +test "HF reports a path absent from the repository listing as FileNotFound" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{}); + defer hf.deinit(); + + // The listing `getOrFetchTree` would keep, so no request is sent. + var tree: std.ArrayList(TreeNode) = .empty; + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "config.json", .type = "file", .size = 2 }); + try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "nested/model.safetensors", .type = "file", .size = 8 }); + try hf.trees.put(allocator, try allocator.dupe(u8, "owner/model@main"), tree); + + const io = hf.io(); + try std.testing.expectEqual(@as(u64, 2), (try std.Io.Dir.statFile(.cwd(), io, "owner/model/config.json", .{})).size); + try std.Io.Dir.access(.cwd(), io, "owner/model/nested/model.safetensors", .{}); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model", .{})); + // Opening a directory as a file sends no request. + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model/nested", .{})); + try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model", .{})); + + try std.testing.expectError(error.FileNotFound, std.Io.Dir.statFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.access(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openFile(.cwd(), io, "owner/model/missing.json", .{})); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.openDir(.cwd(), io, "owner/model/missing", .{})); +} + +test "HF sends its token to the Hub only" { + const allocator = std.testing.allocator; + var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; + defer client.deinit(); + var hf: HF = try .init(allocator, std.testing.io, &client, .{ .token = "hf_token" }); + defer hf.deinit(); + + try std.testing.expectEqualStrings("Bearer hf_token", hf.hubAuthorization("https://huggingface.co/owner/model/resolve/main/model.safetensors").override); + // Not the Hub: the CDNs its resolve URLs redirect to, a look-alike + // host, plain http. + for ([_][]const u8{ + "https://cas-bridge.xethub.hf.co/xet-bridge-us/0123?X-Amz-Signature=abc", + "https://cdn-lfs.huggingface.co/repos/00/11/0123", + "https://huggingface.co.evil.com/owner/model/resolve/main/model.safetensors", + "http://huggingface.co/owner/model/resolve/main/model.safetensors", + }) |url| try std.testing.expect(hf.hubAuthorization(url) == .omit); +} diff --git a/vfs/http.zig b/vfs/http.zig index 9054983e..a9765d0f 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -3,10 +3,18 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { + pub const Options = struct { + protocol: Protocol, + retry: request.Retry = .{}, + }; + const Handle = struct { pub const Type = enum { file, @@ -39,22 +47,23 @@ pub const HTTP = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, protocol: Protocol, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HTTP { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .protocol = protocol, + .protocol = options.protocol, + .governor = .init(inner, http_client, "http", .server_failure, options.retry), }; } pub fn deinit(self: *HTTP) void { + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -94,6 +103,13 @@ pub const HTTP = struct { }; } + pub fn backend(self: *HTTP) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *HTTP) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -144,9 +160,9 @@ pub const HTTP = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -193,7 +209,9 @@ pub const HTTP = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = @intCast(0), @@ -208,15 +226,19 @@ pub const HTTP = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -282,20 +304,18 @@ pub const HTTP = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); + // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos + @as(u64, @intCast(relative_offset)) + handle.pos +| @abs(relative_offset) else - handle.pos - @as(u64, @intCast(-relative_offset)); + handle.pos -| @abs(relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -311,117 +331,31 @@ pub const HTTP = struct { return path.len; } - fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - var aux_buffer: []u8 = &redirect_buffer; + /// The size of the object at `url` (a resolved path without its scheme), + /// from a HEAD of the end of its redirect chain, as far as a read's GET + /// follows it (`request.max_redirects`). + fn fetchSize(self: *HTTP, url: []const u8) !u64 { + var url_buffer: [8 * 1024]u8 = undefined; + const full_url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), url }); + var chain: request.HeadChain = .{}; + const end = try chain.resolve(&self.governor, url, full_url, request.max_redirects, {}, urlAuthorization); + return end.size orelse error.MissingContentLength; + } - const scheme = @tagName(self.protocol); - const url = try self.resolvePath(dir, sub_path, &path_buffer); - const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); - - var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; - while (true) { - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var res = try req.receiveHead(&redirect_buffer); - - switch (res.head.status.class()) { - .server_error, .client_error => { - log.err("Failed to fetch tree size for {s}", .{url}); - log.err("{s}", .{res.head.bytes}); - return error.ServerError; - }, - .informational => return error.UnexpectedStatus, - .success => return res.head.content_length.?, - .redirect => { - const location = res.head.location.?; - @memcpy(aux_buffer[0..location.len], location); - uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; - continue; - }, - } - } + /// Only what the URL carries: std's basic authorization from its + /// userinfo, as on every GET. + fn urlAuthorization(_: void, _: []const u8) std.http.Client.Request.Headers.Value { + return .default; } fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; - var url_buffer: [8 * 1024]u8 = undefined; const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); const uri: std.Uri = try .parse(url); - - var req = try self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{handle.uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } - } - - return try reader.readSliceShort(data[0]); - } - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url, .key = request.authorityOf(uri) }, + // The object `fetchSize` found at the end of the chain. + .redirects = .follow, + }, &uri, range_read.static, data, offset, handle.size); } }; diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig new file mode 100644 index 00000000..a09464a5 --- /dev/null +++ b/vfs/http_acceptance_test.zig @@ -0,0 +1,704 @@ +const std = @import("std"); + +const GCS = @import("gcs.zig").GCS; +const HTTP = @import("http.zig").HTTP; +const MockServer = @import("mock_server.zig").MockServer; +const range_read = @import("range_read.zig"); +const vfs_request = @import("request.zig"); +const S3 = @import("s3.zig").S3; + +/// A retry configuration that never sleeps, for the tests that count +/// attempts rather than time. +fn instantRetries(max_retries: usize) vfs_request.Retry { + return .{ + .max_retries = max_retries, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + .max_hold = .fromNanoseconds(0), + .throttle_budget = .fromSeconds(300), + }; +} + +/// A retry configuration whose retries and holds are short enough for a test. +fn fastRetry(initial_ms: i64, budget_ms: i64) vfs_request.Retry { + return .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(initial_ms), + .max_hold = .fromSeconds(30), + .throttle_budget = .fromMilliseconds(budget_ms), + }; +} + +/// A `MockServer` serving `object`, an HTTP backend with a client of its +/// own, and `/object` open through that backend. Initialized in place: the +/// backend points at the client and the server must not move once started. +const Fixture = struct { + server: MockServer, + client: std.http.Client, + http: HTTP, + file: std.Io.File, + + fn init(self: *Fixture, object: []const u8, server_opts: MockServer.Options, retry: vfs_request.Retry) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + self.server = try .init(io, object, server_opts); + errdefer self.server.deinit(io); + try self.server.start(io); + self.client = .{ .allocator = allocator, .io = io }; + errdefer self.client.deinit(); + self.http = try .init(allocator, io, &self.client, .{ .protocol = .http, .retry = retry }); + errdefer self.http.deinit(); + var path_buffer: [128]u8 = undefined; + const path = try self.server.path(&path_buffer); + self.file = try std.Io.Dir.openFile(.cwd(), self.http.io(), path, .{ .mode = .read_only }); + } + + fn deinit(self: *Fixture) void { + self.file.close(self.http.io()); + self.http.deinit(); + self.client.deinit(); + self.server.deinit(std.testing.io); + } +}; + +/// Reads `object` back through the fixture's file with `readers` concurrent +/// positional reads of `read_size` bytes, reader `i` at `i * read_size`, +/// all released at once. Fails on the first reader that failed or read +/// other bytes. +fn expectConcurrentReads(fixture: *Fixture, object: []const u8, comptime readers: usize, comptime read_size: usize) !void { + const Reader = struct { + fn run(file: std.Io.File, file_io: std.Io, output: []u8, offset: u64, result: *anyerror!usize, start: *std.Io.Event) void { + start.wait(file_io) catch |err| { + result.* = err; + return; + }; + result.* = file.readPositional(file_io, &.{output}, offset); + } + }; + const io = std.testing.io; + var outputs: [readers][read_size]u8 = undefined; + var results: [readers]anyerror!usize = undefined; + var released: std.Io.Event = .unset; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (&outputs, &results, 0..) |*output, *result, index| { + try group.concurrent(io, Reader.run, .{ fixture.file, fixture.http.io(), output, index * read_size, result, &released }); + } + released.set(io); + try group.await(io); + + for (&outputs, results, 0..) |*output, result, index| { + try std.testing.expectEqual(read_size, try result); + try std.testing.expectEqualSlices(u8, object[index * read_size ..][0..read_size], output); + } +} + +test "generic HTTP keeps a large scattered positional read to one GET" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const offset = 13; + const request_size = 17 * 1024 * 1024 + 257; + + const object = try allocator.alloc(u8, offset + request_size); + defer allocator.free(object); + @memset(object, 0xa5); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{}, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + // Fresh stats for the read: the open's HEAD is a request too. + stats.* = .{}; + const buffers = [_][]u8{ + output[0 .. 4 * 1024 * 1024], + output[4 * 1024 * 1024 .. 10 * 1024 * 1024], + output[10 * 1024 * 1024 ..], + }; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(fixture.http.io(), &buffers, offset), + ); + try std.testing.expectEqualSlices(u8, object[offset..], output); + + try server.stop(io); + + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} + +test "generic HTTP follows a relative redirect chain at open and on every read" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // `/object` redirects twice, and the second location resolves against + // the first hop's path. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .redirects = &.{ + .{ .path = "/object", .location = "/hop/next" }, + .{ .path = "/hop/next", .location = "../moved" }, + }, + .objects = &.{.{ .path = "/moved", .bytes = &object }}, + }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + + try std.testing.expectEqual(@as(u64, object.len), (try fixture.file.stat(http_io)).size); + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(http_io, &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + + try server.stop(io); + // Only the object answers with bytes; the hops before it redirect. + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); +} + +test "generic HTTP retries serially and reports aggregate retry counters" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 2 * 1024 * 1024; + + const object = try allocator.alloc(u8, request_size); + defer allocator.free(object); + @memset(object, 0x3c); + const output = try allocator.alloc(u8, request_size); + defer allocator.free(output); + + var fixture: Fixture = undefined; + try fixture.init(object, .{ .fail_first_gets = 1 }, .{ + .max_retries = 1, + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + }); + defer fixture.deinit(); + const server = &fixture.server; + const http_io = fixture.http.io(); + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const one_buffer = [_][]u8{output}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + + stats.* = .{}; + try std.testing.expectEqual( + request_size, + try fixture.file.readPositional(http_io, &one_buffer, 0), + ); + try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqualSlices(u8, object, output); + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); +} + +test "generic HTTP physical concurrency does not exceed caller admission" { + const io = std.testing.io; + const admission = 3; + const read_size = 1024; + + var object: [admission * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = admission }, .{}); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + try expectConcurrentReads(&fixture, &object, admission, read_size); + + try server.stop(io); + + const peak = server.peak_gets.load(.acquire); + try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, admission), peak); + try std.testing.expect(peak <= admission); + try std.testing.expectEqual( + @as(usize, admission), + server.get_requests.load(.acquire), + ); + try std.testing.expectEqual(@as(u64, admission), stats.physical_requests.load(.acquire)); +} + +test "the shared range loop prepares the request once per attempt" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const request_size = 3 * 1024; + + var object: [request_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 7); + var output: [request_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ .fail_first_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + // An S3-style request: as a SigV4 signature must be, it is prepared + // again by the hook for every attempt. + const Signed = struct { + uri: std.Uri, + prepared: usize = 0, + + fn prepare(self: *@This(), _: vfs_request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + try std.testing.expectEqualStrings("Range", range.name); + self.prepared += 1; + return .{ .uri = self.uri }; + } + }; + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + var signed: Signed = .{ .uri = try .parse(url) }; + const spec: range_read.RangeSpec = .{ + .request = .{ .target = url, .key = vfs_request.authorityOf(signed.uri) }, + }; + const buffers = [_][]u8{ output[0..1024], output[1024..] }; + + // No retry budget: one attempt, one hook call, the failure is counted. + var no_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(0)); + try std.testing.expectError( + error.RetriesExhausted, + range_read.performRangeRead(&no_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 1), signed.prepared); + try std.testing.expectEqual(@as(u64, 1), no_retry.stats.server_failures.load(.acquire)); + + signed.prepared = 0; + var one_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(1)); + try std.testing.expectEqual( + object.len, + try range_read.performRangeRead(&one_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), + ); + try std.testing.expectEqual(@as(usize, 2), signed.prepared); + try std.testing.expectEqualSlices(u8, &object, &output); + + try server.stop(io); + + // One GET per hook call. + try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); + const stats = &one_retry.stats; + try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); +} + +test "one throttled GET holds every reader of the backend" { + const io = std.testing.io; + const readers = 8; + const read_size = 512; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index); + + // Every reader's first GET is throttled, all eight answered together (the + // barrier), with a one second reset. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .barrier_gets = readers, + .throttle = .{ .first_gets = readers, .retry_after_s = 1 }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try server.stop(io); + + // One hold covered every reader, the server-named second was honoured, + // and no reader burned a retry for it. + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, readers), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(usize, 2 * readers), server.get_requests.load(.acquire)); + const throttled_at = server.first_throttle_ns.load(.acquire); + const resumed_at = server.first_get_after_throttle_ns.load(.acquire); + try std.testing.expect(throttled_at != 0 and resumed_at != 0); + try std.testing.expect(resumed_at - throttled_at >= std.time.ns_per_s); +} + +test "a rate window is absorbed by the hold instead of the retry budget" { + const io = std.testing.io; + const readers = 12; + const read_size = 256; + + var object: [readers * read_size]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 3); + + // Four GETs per 100 ms; twelve readers need three windows at best. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .window = .{ .gets = 4, .per_ms = 100 } }, + }, fastRetry(50, 300_000)); + defer fixture.deinit(); + const stats = &fixture.http.governor.stats; + + try expectConcurrentReads(&fixture, &object, readers, read_size); + + try fixture.server.stop(io); + + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.throttles.load(.acquire) > 0); + // A limited server never exhausts a reader's retries: the hold carries + // the whole episode. + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a throttled HEAD at open waits the hold out and succeeds" { + const io = std.testing.io; + + var object: [64]u8 = undefined; + @memset(&object, 0x11); + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_heads = 1, .retry_after_s = 0 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + try std.testing.expectEqual(@as(u64, object.len), try fixture.file.length(fixture.http.io())); + + try server.stop(io); + + // The open retried its HEAD after the hold, and the hold is the + // backend's, not the read path's. + try std.testing.expectEqual(@as(usize, 2), server.head_requests.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); +} + +test "a 503 holds an object store and only retries a plain HTTP server" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 512; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x7e); + var output: [read_size]u8 = undefined; + + // Each backend's first two GETs are answered 503. + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .first_gets = 2, .status = .service_unavailable }, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(5) }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(5) }); + defer gcs.deinit(); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); + defer http.deinit(); + + var path_buffer: [128]u8 = undefined; + const backends = [_]struct { std.Io, *const vfs_request.Governor, []const u8, bool }{ + .{ s3.io(), &s3.governor, "bucket/object", true }, + .{ gcs.io(), &gcs.governor, "bucket/object", true }, + .{ http.io(), &http.governor, try server.path(&path_buffer), false }, + }; + for (backends) |backend| { + const backend_io, const governor, const path, const object_store = backend; + const file = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + defer file.close(backend_io); + server.get_requests.store(0, .release); + try std.testing.expectEqual(read_size, try file.readPositional(backend_io, &.{&output}, 0)); + + const stats = &governor.stats; + if (object_store) { + // A 503 is rate limiting. One reader, two throttles in a row: + // each arms its own hold once the previous one has passed, and + // neither is charged a retry. + try std.testing.expectEqual(@as(u64, 2), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + } else { + // A 503 is a server failure: no hold, one charged retry per + // attempt. + try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); + } + } + + try server.stop(io); +} + +test "a server that never lets up fails the request with RateLimited" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x33); + var output: [read_size]u8 = undefined; + + var server = try MockServer.init(io, &object, .{ + .throttle = .{ .always = true, .retry_after_s = 0 }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var url_buffer: [160]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const uri: std.Uri = try .parse(url); + const buffers = [_][]u8{&output}; + + var governor: vfs_request.Governor = .init(io, &client, "test", .throttle, .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromMilliseconds(50), + .throttle_budget = .fromMilliseconds(300), + }); + const started: std.Io.Timestamp = .now(io, .awake); + // The read never runs: the backend is rate limited past its budget. + // `fileReadPositional` maps this to `Unexpected` with its own log line. + try std.testing.expectError(error.RateLimited, range_read.performRangeRead(&governor, .{ + .request = .{ .target = url }, + }, &uri, range_read.static, &buffers, 0, object.len)); + const elapsed = started.durationTo(.now(io, .awake)).nanoseconds; + try std.testing.expect(elapsed >= 300 * std.time.ns_per_ms); + try std.testing.expect(elapsed < 5 * std.time.ns_per_s); + try std.testing.expect(governor.stats.holds.load(.acquire) > 1); + + try server.stop(io); +} + +test "a reader cancelled while the backend is held returns Canceled" { + const io = std.testing.io; + const read_size = 256; + + var object: [read_size]u8 = undefined; + @memset(&object, 0x21); + var output: [read_size]u8 = undefined; + + var fixture: Fixture = undefined; + try fixture.init(&object, .{ + .throttle = .{ .first_gets = 1, .retry_after_s = 30 }, + }, fastRetry(20, 300_000)); + defer fixture.deinit(); + + var failure: std.atomic.Value(u16) = .init(0); + var readers: std.Io.Group = .init; + try readers.concurrent(io, struct { + fn run(file_: std.Io.File, io_: std.Io, out: []u8, failure_: *std.atomic.Value(u16)) void { + const buffers = [_][]u8{out}; + _ = file_.readPositional(io_, &buffers, 0) catch |err| { + failure_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ fixture.file, fixture.http.io(), output[0..], &failure }); + + // Let the first GET be throttled and the reader settle into the hold. + try io.sleep(.fromMilliseconds(50), .awake); + readers.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), failure.load(.acquire)); + + try fixture.server.stop(io); +} + +test "generic HTTP cancels a GET that receives nothing and retries it as a timeout" { + const io = std.testing.io; + + var object: [4096]u8 = undefined; + for (&object, 0..) |*byte, i| byte.* = @truncate(i); + var output: [1000]u8 = undefined; + + // The first GET waits unanswered for the second: a peer that receives + // the request and sends nothing. + var fixture: Fixture = undefined; + try fixture.init(&object, .{ .barrier_gets = 2 }, .{ + .initial_delay = .fromNanoseconds(0), + .max_delay = .fromNanoseconds(0), + .stall_timeout = .fromMilliseconds(100), + }); + defer fixture.deinit(); + const server = &fixture.server; + const stats = &fixture.http.governor.stats; + + stats.* = .{}; + const buffers = [_][]u8{&output}; + try std.testing.expectEqual(output.len, try fixture.file.readPositional(fixture.http.io(), &buffers, 77)); + try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); + try std.testing.expectEqual(@as(u64, 1), stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); + + // The first GET answers late, to a connection the client closed: that + // write may fail. + server.stop(io) catch |err| if (err != error.WriteFailed) return err; + try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); +} + +test "http, s3 and gs fail open, stat and access of an absent or denied object alike" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Case = struct { name: []const u8, status: std.http.Status, expected: anyerror }; + const cases = [_]Case{ + .{ .name = "missing", .status = .not_found, .expected = error.FileNotFound }, + .{ .name = "gone", .status = .gone, .expected = error.FileNotFound }, + .{ .name = "unauthorized", .status = .unauthorized, .expected = error.PermissionDenied }, + .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, + }; + // The target each backend sends for an object: HTTP its path, S3 + // `//`, GCS the same with `?alt=media`. + const targets = [_]struct { []const u8, []const u8 }{ .{ "/", "" }, .{ "/bucket/", "" }, .{ "/bucket/", "?alt=media" } }; + const refusals = comptime refusals: { + var list: [cases.len * targets.len]MockServer.Refusal = undefined; + for (cases, 0..) |case, i| { + for (targets, 0..) |target, j| { + list[i * targets.len + j] = .{ .path = target[0] ++ case.name ++ target[1], .status = case.status }; + } + } + break :refusals list; + }; + + var object: [32]u8 = undefined; + @memset(&object, 0x6b); + var server = try MockServer.init(io, &object, .{ + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + .refusals = &refusals, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint }); + defer gcs.deinit(); + + var authority_buffer: [64]u8 = undefined; + const backends = [_]struct { std.Io, []const u8 }{ + .{ http.io(), try std.fmt.bufPrint(&authority_buffer, "{s}/", .{endpoint["http://".len..]}) }, + .{ s3.io(), "bucket/" }, + .{ gcs.io(), "bucket/" }, + }; + for (backends) |backend| { + const backend_io, const prefix = backend; + var path_buffer: [128]u8 = undefined; + const present = try std.fmt.bufPrint(&path_buffer, "{s}object", .{prefix}); + try std.Io.Dir.access(.cwd(), backend_io, present, .{}); + try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), backend_io, present, .{})).size); + for (cases) |case| { + const path = try std.fmt.bufPrint(&path_buffer, "{s}{s}", .{ prefix, case.name }); + try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), backend_io, path, .{})); + try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), backend_io, path, .{})); + } + } + + try server.stop(io); +} + +test "http, s3 and gs fail a refused streaming read with InputOutput and seek back no further than 0" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var object: [64]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); + // Each backend's first GET is answered 500, and none has a retry left. + var server = try MockServer.init(io, &object, .{ + .fail_first_gets = 3, + .objects = &.{ + .{ .path = "/bucket/object", .bytes = &object }, + .{ .path = "/bucket/object?alt=media", .bytes = &object }, + }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(0) }); + defer http.deinit(); + var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(0) }); + defer s3.deinit(); + var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(0) }); + defer gcs.deinit(); + + var path_buffer: [128]u8 = undefined; + const backend_ios = [_]std.Io{ http.io(), s3.io(), gcs.io() }; + const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object", "bucket/object" }; + var files: [backend_ios.len]std.Io.File = undefined; + var opened: usize = 0; + defer for (files[0..opened], backend_ios[0..opened]) |file, backend_io| file.close(backend_io); + for (&files, backend_ios, paths) |*file, backend_io, path| { + file.* = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); + opened += 1; + } + + var head: [8]u8 = undefined; + const head_slices = [_][]u8{&head}; + for (files, backend_ios) |file, backend_io| { + try std.testing.expectError(error.InputOutput, file.readStreaming(backend_io, &head_slices)); + } + for (files, backend_ios) |file, backend_io| { + // Forward, then back by more than the position: at 0. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, 5); + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, -9); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + // `minInt(i64)`, whose negation overflows, stops at 0 too. + try backend_io.vtable.fileSeekBy(backend_io.userdata, file, std.math.minInt(i64)); + try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); + try std.testing.expectEqualSlices(u8, object[0..head.len], &head); + } + + try server.stop(io); + try std.testing.expectEqual(@as(usize, 9), server.get_requests.load(.acquire)); +} diff --git a/vfs/mock_server.zig b/vfs/mock_server.zig new file mode 100644 index 00000000..9206f65b --- /dev/null +++ b/vfs/mock_server.zig @@ -0,0 +1,326 @@ +//! A minimal HTTP origin for the VFS and loader tests: one object per path, +//! range reads, and injectable failures and rate limiting. Exported as +//! `VFS.MockServer` so tests above this layer can serve a fixture over a +//! real socket; nothing outside a test build references it. + +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const MockServer = struct { + /// How the server rate limits. A throttled answer carries the status + /// and, when the test names one, a `Retry-After`. + const Throttle = struct { + /// The first N GETs are throttled. + first_gets: usize = 0, + /// The first N HEADs are throttled. + first_heads: usize = 0, + /// Every GET is throttled, forever. + always: bool = false, + status: std.http.Status = .too_many_requests, + retry_after_s: ?u32 = null, + /// At most `gets` GETs per `per_ms` milliseconds; the rest are + /// throttled. + window: ?struct { gets: usize, per_ms: i64 } = null, + }; + + /// One extra path the server serves beside `/object`. + const Object = struct { path: []const u8, bytes: []const u8 }; + + /// A path answered with `status` and no object, on purpose. + pub const Refusal = struct { path: []const u8, status: std.http.Status }; + + /// A path answered with a 302 to `location`, for HEAD and GET alike. + pub const Redirect = struct { path: []const u8, location: []const u8 }; + + pub const Options = struct { + fail_first_gets: usize = 0, + barrier_gets: usize = 0, + throttle: Throttle = .{}, + objects: []const Object = &.{}, + /// Paths answered with a status of their own (404, 403, ...). + refusals: []const Refusal = &.{}, + redirects: []const Redirect = &.{}, + }; + + tcp: std.Io.net.Server, + /// The accept loop and its connections, from `start` to `stop`. + group: std.Io.Group = .init, + object: []const u8, + fail_first_gets: usize, + barrier_gets: usize, + throttle: Throttle, + objects: []const Object, + refusals: []const Refusal, + redirects: []const Redirect, + get_barrier: std.Io.Event = .unset, + head_requests: std.atomic.Value(usize) = .init(0), + get_requests: std.atomic.Value(usize) = .init(0), + active_gets: std.atomic.Value(usize) = .init(0), + peak_gets: std.atomic.Value(usize) = .init(0), + first_error: stdx.FirstError(anyerror) = .{}, + /// Awake nanoseconds of the first throttled answer and of the first + /// answer served after one; zero until they happen. + first_throttle_ns: std.atomic.Value(i64) = .init(0), + first_get_after_throttle_ns: std.atomic.Value(i64) = .init(0), + /// Guards the rate window. + window_mutex: std.Io.Mutex = .init, + window_started_ns: i64 = 0, + window_gets: usize = 0, + + pub fn init( + io: std.Io, + object: []const u8, + opts: Options, + ) !MockServer { + const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; + return .{ + .tcp = try address.listen(io, .{ .reuse_address = true }), + .object = object, + .fail_first_gets = opts.fail_first_gets, + .barrier_gets = opts.barrier_gets, + .throttle = opts.throttle, + .objects = opts.objects, + .refusals = opts.refusals, + .redirects = opts.redirects, + }; + } + + /// Serves on a task of its own until `stop`. The server must not move + /// from here on. + pub fn start(self: *MockServer, io: std.Io) !void { + try self.group.concurrent(io, run, .{ self, io }); + } + + /// Stops serving, each connection canceled and joined, and returns the + /// first error the server met. + pub fn stop(self: *MockServer, io: std.Io) !void { + self.group.cancel(io); + if (self.first_error.get()) |err| return err; + } + + /// Stops serving if `stop` did not, and closes the listening socket. + pub fn deinit(self: *MockServer, io: std.Io) void { + self.group.cancel(io); + self.tcp.deinit(io); + } + + /// The path of `/object` for an `http` backend: `127.0.0.1:/object`. + pub fn path(self: *const MockServer, buffer: []u8) ![]const u8 { + return std.fmt.bufPrint(buffer, "127.0.0.1:{d}/object", .{self.port()}); + } + + fn bodyFor(self: *const MockServer, target: []const u8) ?[]const u8 { + if (std.mem.eql(u8, target, "/object")) return self.object; + for (self.objects) |object| { + if (std.mem.eql(u8, object.path, target)) return object.bytes; + } + return null; + } + + fn refusalFor(self: *const MockServer, target: []const u8) ?std.http.Status { + for (self.refusals) |refusal| { + if (std.mem.eql(u8, refusal.path, target)) return refusal.status; + } + return null; + } + + fn redirectFor(self: *const MockServer, target: []const u8) ?[]const u8 { + for (self.redirects) |redirect| { + if (std.mem.eql(u8, redirect.path, target)) return redirect.location; + } + return null; + } + + fn nowNs(io: std.Io) i64 { + return @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + } + + /// Whether this GET is over the rate window. Counts only the requests + /// it lets through, so a throttled one does not consume the budget. + fn overWindow(self: *MockServer, io: std.Io) bool { + const window = self.throttle.window orelse return false; + self.window_mutex.lockUncancelable(io); + defer self.window_mutex.unlock(io); + const now = nowNs(io); + if (now - self.window_started_ns > window.per_ms * std.time.ns_per_ms) { + self.window_started_ns = now; + self.window_gets = 0; + } + if (self.window_gets >= window.gets) return true; + self.window_gets += 1; + return false; + } + + fn respondThrottled(self: *MockServer, io: std.Io, request: *std.http.Server.Request) !void { + _ = self.first_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + var retry_after_buffer: [16]u8 = undefined; + try request.respond("slow down", .{ + .status = self.throttle.status, + .keep_alive = false, + .extra_headers = if (self.throttle.retry_after_s) |seconds| &.{.{ + .name = "Retry-After", + .value = try std.fmt.bufPrint(&retry_after_buffer, "{d}", .{seconds}), + }} else &.{}, + }); + } + + pub fn port(self: *const MockServer) u16 { + return self.tcp.socket.address.getPort(); + } + + fn run(self: *MockServer, io: std.Io) std.Io.Cancelable!void { + var connections: std.Io.Group = .init; + defer connections.cancel(io); + + while (true) { + const stream = self.tcp.accept(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + _ = self.first_error.set(err); + return; + }, + }; + connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { + stream.close(io); + _ = self.first_error.set(err); + return; + }; + } + } + + fn onConnection( + self: *MockServer, + stream: std.Io.net.Stream, + io: std.Io, + ) std.Io.Cancelable!void { + defer stream.close(io); + self.handleConnection(stream, io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => _ = self.first_error.set(err), + }; + } + + fn handleConnection(self: *MockServer, stream: std.Io.net.Stream, io: std.Io) !void { + var read_buffer: [8 * 1024]u8 = undefined; + var reader = stream.reader(io, &read_buffer); + var write_buffer: [8 * 1024]u8 = undefined; + var writer = stream.writer(io, &write_buffer); + var server: std.http.Server = .init(&reader.interface, &writer.interface); + var request = try server.receiveHead(); + + if (self.redirectFor(request.head.target)) |location| { + try request.respond("", .{ + .status = .found, + .keep_alive = false, + .extra_headers = &.{.{ .name = "Location", .value = location }}, + }); + return; + } + + const body = self.bodyFor(request.head.target) orelse { + const refusal = self.refusalFor(request.head.target); + try request.respond("refused", .{ + .status = refusal orelse .not_found, + .keep_alive = false, + }); + if (refusal != null) return; + return error.UnexpectedTarget; + }; + + switch (request.head.method) { + .HEAD => { + const head_ordinal = self.head_requests.fetchAdd(1, .acq_rel) + 1; + if (head_ordinal <= self.throttle.first_heads) { + try self.respondThrottled(io, &request); + return; + } + try request.respond(body, .{ .keep_alive = false }); + }, + .GET => { + const range = try parseRequestRange(&request, body.len); + const get_ordinal = self.get_requests.fetchAdd(1, .acq_rel) + 1; + const active = self.active_gets.fetchAdd(1, .acq_rel) + 1; + _ = self.peak_gets.fetchMax(active, .acq_rel); + defer _ = self.active_gets.fetchSub(1, .release); + + if (self.barrier_gets != 0 and get_ordinal <= self.barrier_gets) { + if (get_ordinal == self.barrier_gets) self.get_barrier.set(io); + try self.get_barrier.wait(io); + } + + if (get_ordinal <= self.fail_first_gets) { + try request.respond("retry", .{ + .status = .internal_server_error, + .keep_alive = false, + }); + return; + } + + if (self.throttle.always or get_ordinal <= self.throttle.first_gets or self.overWindow(io)) { + try self.respondThrottled(io, &request); + return; + } + if (self.first_throttle_ns.load(.acquire) != 0) { + _ = self.first_get_after_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); + } + + var content_range_buffer: [96]u8 = undefined; + const content_range = try std.fmt.bufPrint( + &content_range_buffer, + "bytes {d}-{d}/{d}", + .{ range.start, range.end, body.len }, + ); + try request.respond(body[range.start .. range.end + 1], .{ + .status = .partial_content, + .keep_alive = false, + .extra_headers = &.{ + .{ .name = "Content-Range", .value = content_range }, + }, + }); + }, + else => { + try request.respond("method not allowed", .{ + .status = .method_not_allowed, + .keep_alive = false, + }); + return error.UnexpectedMethod; + }, + } + } + + const Range = struct { + start: usize, + end: usize, + }; + + fn parseRequestRange(request: *const std.http.Server.Request, object_len: usize) !Range { + var value: ?[]const u8 = null; + var headers = request.iterateHeaders(); + while (headers.next()) |header| { + if (!std.ascii.eqlIgnoreCase(header.name, "Range")) continue; + if (value != null) return error.DuplicateRange; + value = header.value; + } + + const range_value = value orelse return error.MissingRange; + const prefix = "bytes="; + if (range_value.len <= prefix.len or + !std.ascii.eqlIgnoreCase(range_value[0..prefix.len], prefix)) + { + return error.InvalidRange; + } + const dash = std.mem.indexOfScalar(u8, range_value[prefix.len..], '-') orelse + return error.InvalidRange; + const absolute_dash = prefix.len + dash; + if (absolute_dash == prefix.len or absolute_dash + 1 == range_value.len) + return error.InvalidRange; + + const result: Range = .{ + .start = try std.fmt.parseInt(usize, range_value[prefix.len..absolute_dash], 10), + .end = try std.fmt.parseInt(usize, range_value[absolute_dash + 1 ..], 10), + }; + if (result.end < result.start or result.end >= object_len) return error.InvalidRange; + return result; + } +}; diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig deleted file mode 100644 index 70c9d89d..00000000 --- a/vfs/parallel_read.zig +++ /dev/null @@ -1,211 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const InitOpts = struct { - chunk_size: usize, - num_workers: usize, - queue_capacity: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, -}; - -pub const Status = union(enum) { - success, - retry_after: ?std.Io.Duration, - - pub fn retry() Status { - return .{ .retry_after = null }; - } -}; - -pub const BatchState = struct { - pending: std.atomic.Value(u32), - err: std.atomic.Value(u16) = .init(0), - - pub fn failed(batch: *BatchState) bool { - return batch.err.load(.acquire) != 0; - } - - pub fn fail(batch: *BatchState, err: anyerror) void { - _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); - } - - pub fn anyError(batch: *BatchState) ?anyerror { - const err = batch.err.load(.acquire); - return if (err == 0) null else @errorFromInt(err); - } - - pub fn completeOne(batch: *BatchState, io: std.Io) void { - const previous = batch.pending.fetchSub(1, .release); - std.debug.assert(previous > 0); - if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); - } - - pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { - while (true) { - const pending = batch.pending.load(.acquire); - if (pending == 0) return; - io.futexWaitUncancelable(u32, &batch.pending.raw, pending); - } - } -}; - -pub fn Pool(comptime Job: type) type { - return struct { - const Self = @This(); - - group: std.Io.Group = .init, - job_queue: std.Io.Queue(Job), - queue_buf: []Job, - client: *std.http.Client, - prng: std.Random.DefaultPrng, - chunk_size: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, - - pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { - stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); - stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); - stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); - stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); - - const queue_buf = try allocator.alloc(Job, opts.queue_capacity); - errdefer allocator.free(queue_buf); - - self.* = .{ - .client = client, - .chunk_size = opts.chunk_size, - .max_retries = opts.max_retries, - .retry_initial_delay = opts.retry_initial_delay, - .retry_max_delay = opts.retry_max_delay, - .prng = .init(blk: { - var seed: u64 = undefined; - io.random(@ptrCast(&seed)); - break :blk seed; - }), - .job_queue = .init(queue_buf), - .queue_buf = queue_buf, - }; - errdefer self.group.cancel(io); - - for (0..opts.num_workers) |_| { - try self.group.concurrent(io, worker, .{ self, io }); - } - } - - pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { - self.job_queue.close(io); - self.group.cancel(io); - - allocator.free(self.queue_buf); - } - - fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { - while (true) { - const job = pool.job_queue.getOne(io) catch break; - - var attempt: usize = 0; - while (true) { - if (job.batch.state.failed()) break; - const status = job.perform(pool.client) catch |err| { - job.batch.state.fail(err); - break; - }; - - switch (status) { - .success => break, - .retry_after => |duration| { - if (attempt >= pool.max_retries) { - job.batch.state.fail(error.RetriesExhausted); - break; - } - - io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { - job.batch.state.fail(error.RetriesExhausted); - break; - }; - - attempt += 1; - }, - } - continue; - } - - job.batch.state.completeOne(io); - } - } - - fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { - // FullJitter algorithm - const max_delay_ns: i96 = @min( - pool.retry_max_delay.toNanoseconds(), - pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), - ); - const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); - return .fromNanoseconds(delay_ns); - } - }; -} - -pub const ContentRange = struct { - start: u64, - end: u64, - total: u64, -}; - -pub fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; -} - -pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { - if (offset >= file_size) return 0; - - var requested: usize = 0; - for (data) |buf| requested += buf.len; - return @intCast(@min(file_size - offset, requested)); -} - -pub fn readChunk( - reader: *std.Io.Reader, - content_range: ?ContentRange, - file_offset: u64, - data: []const []u8, - chunk_offset: usize, - chunk_len: usize, -) !void { - if (content_range) |cr| { - if (cr.start > file_offset) return error.InvalidContentRange; - if (cr.start < file_offset) { - try reader.discardAll(file_offset - cr.start); - } - } - - var remaining = chunk_len; - var skip = chunk_offset; - - for (data) |buf| { - if (skip >= buf.len) { - skip -= buf.len; - continue; - } - - const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; - try reader.readSliceAll(destination); - remaining -= destination.len; - if (remaining == 0) break; - skip = 0; - } -} diff --git a/vfs/range_read.zig b/vfs/range_read.zig new file mode 100644 index 00000000..920720e6 --- /dev/null +++ b/vfs/range_read.zig @@ -0,0 +1,263 @@ +//! The Range specifics of a governed request: the `Range` header, the +//! `Content-Range` validation and the scatter into the caller's buffers. +//! Everything else (the hold, the retries, the classification) is in +//! `request.zig`. + +const std = @import("std"); + +const request = @import("request.zig"); + +const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +/// The backend's part of one GET: the URI, the authorization value and up to +/// `max_extra_headers` headers. The loop appends `Range`. Every slice must +/// stay valid until the attempt completes, so hooks write into storage that +/// outlives the call (the backend's per-read context). +pub const PreparedRequest = struct { + uri: std.Uri, + authorization: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, +}; + +pub const max_extra_headers = 4; + +/// One range read: what identifies it, and whether it follows a redirect. +pub const RangeSpec = struct { + request: request.RequestSpec, + /// The object stores and a resolved download URL never redirect a + /// read; a plain HTTP server may. + redirects: request.Redirects = .forbid, +}; + +/// The `prepare` of a request that is the same on every attempt: a plain +/// GET of `uri`. +pub fn static(uri: *const std.Uri, _: request.Attempt, _: std.http.Header) anyerror!PreparedRequest { + return .{ .uri = uri.* }; +} + +/// One `GET` with `Range: bytes=offset-(offset+size-1)` per attempt through +/// the governed loop, where `size` is what `data` holds of the `file_size` +/// bytes past `offset`; nothing is sent when it is 0. `prepare` is called +/// once per attempt: S3's SigV4 signature covers the `Range` header and a +/// timestamp, a bearer token may have expired during a hold, and a signed +/// download URL may need re-resolving after a 403. +/// The caller holds its read slot through every retry, hold and sleep; +/// backend-local parallel readers would multiply the caller's width and +/// defeat its memory bounds. +pub fn performRangeRead( + governor: *request.Governor, + spec: RangeSpec, + context: anytype, + comptime prepare: fn (@TypeOf(context), request.Attempt, std.http.Header) anyerror!PreparedRequest, + data: []const []u8, + offset: u64, + file_size: u64, +) !usize { + const size = readSize(file_size, offset, data); + if (size == 0) return 0; + + const RangeRead = struct { + context: @TypeOf(context), + redirects: request.Redirects, + range: std.http.Header, + data: []const []u8, + offset: u64, + size: usize, + progress: *std.atomic.Value(u64) = undefined, + + fn attempt(self: *@This(), current: request.Attempt) anyerror!request.Outcome(void) { + self.progress = current.progress; + const prepared = try prepare(self.context, current, self.range); + std.debug.assert(prepared.extra_headers.len <= max_extra_headers); + var headers: [max_extra_headers + 1]std.http.Header = undefined; + @memcpy(headers[0..prepared.extra_headers.len], prepared.extra_headers); + headers[prepared.extra_headers.len] = self.range; + return request.exchange(void, current, prepared.uri, .{ + .authorization = prepared.authorization, + .extra_headers = headers[0 .. prepared.extra_headers.len + 1], + .redirects = self.redirects, + }, self, consume); + } + + fn consume(self: *@This(), res: *std.http.Client.Response) anyerror!void { + // The head bytes are released when the body reader is taken. + const content_range = contentRange(res.head); + return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size, self.progress); + } + }; + + var range_buffer: [64]u8 = undefined; + var state: RangeRead = .{ + .context = context, + .redirects = spec.redirects, + .range = .{ + .name = "Range", + .value = std.fmt.bufPrint( + &range_buffer, + "bytes={d}-{d}", + .{ offset, offset + @as(u64, @intCast(size - 1)) }, + ) catch unreachable, + }, + .data = data, + .offset = offset, + .size = size, + }; + try request.perform(void, governor, spec.request, &state, RangeRead.attempt); + governor.stats.recordSuccess(size); + return size; +} + +fn contentRange(head: std.http.Client.Response.Head) ?ContentRange { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) return parseContentRange(header.value); + } + return null; +} + +fn parseContentRange(value: []const u8) ?ContentRange { + const prefix = "bytes "; + if (value.len < prefix.len or !std.ascii.eqlIgnoreCase(value[0..prefix.len], prefix)) return null; + const range_and_total = value[prefix.len..]; + const dash = std.mem.indexOfScalar(u8, range_and_total, '-') orelse return null; + const slash = dash + 1 + (std.mem.indexOfScalar(u8, range_and_total[dash + 1 ..], '/') orelse return null); + if (dash == 0 or slash == dash + 1 or slash + 1 == range_and_total.len) return null; + if (std.mem.indexOfScalar(u8, range_and_total[slash + 1 ..], '/') != null) return null; + + const result: ContentRange = .{ + .start = std.fmt.parseInt(u64, range_and_total[0..dash], 10) catch return null, + .end = std.fmt.parseInt(u64, range_and_total[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, range_and_total[slash + 1 ..], 10) catch return null, + }; + if (result.end < result.start or result.end >= result.total) return null; + return result; +} + +fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested +|= buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +/// Validates the response against the requested range, discards the prefix +/// of a `200` that ignored `Range`, then scatters `read_size` bytes, counting +/// what it receives in `progress`. +fn readResponse( + reader: *std.Io.Reader, + status: std.http.Status, + content_range: ?ContentRange, + offset: u64, + data: []const []u8, + read_size: usize, + progress: ?*std.atomic.Value(u64), +) !void { + std.debug.assert(read_size != 0); + + const response_start = switch (status) { + .partial_content => blk: { + const cr = content_range orelse return error.InvalidContentRange; + const requested_end = std.math.add(u64, offset, read_size - 1) catch return error.InvalidContentRange; + if (cr.start > offset or cr.end < requested_end) return error.InvalidContentRange; + break :blk cr.start; + }, + .ok => 0, + else => return error.UnexpectedStatus, + }; + + var skip = offset - response_start; + while (skip != 0) { + const len = @min(skip, progress_slice); + try reader.discardAll(len); + count(progress, len); + skip -= len; + } + try readScatter(reader, data, read_size, progress); +} + +/// Bytes read between two progress counts. +const progress_slice = 1 << 20; + +fn count(progress: ?*std.atomic.Value(u64), len: u64) void { + if (progress) |counter| _ = counter.fetchAdd(len, .monotonic); +} + +fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize, progress: ?*std.atomic.Value(u64)) !void { + var remaining = len; + for (data) |buf| { + var destination = buf[0..@min(remaining, buf.len)]; + remaining -= destination.len; + while (destination.len != 0) { + const slice = destination[0..@min(destination.len, progress_slice)]; + try reader.readSliceAll(slice); + count(progress, slice.len); + destination = destination[slice.len..]; + } + } + if (remaining != 0) return error.UnexpectedEndOfOutput; +} + +test "Content-Range parsing is strict" { + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("bytes 2-9/10").?, + ); + try std.testing.expectEqual( + ContentRange{ .start = 2, .end = 9, .total = 10 }, + parseContentRange("ByTeS 2-9/10").?, + ); + try std.testing.expect(parseContentRange("items 2-9/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/*") == null); + try std.testing.expect(parseContentRange("bytes 9-2/10") == null); + try std.testing.expect(parseContentRange("bytes 2-10/10") == null); + try std.testing.expect(parseContentRange("bytes 2-9/10 trailing") == null); +} + +test "range responses fill scatter buffers" { + var reader: std.Io.Reader = .fixed("23456789"); + var first: [2]u8 = undefined; + var second: [3]u8 = undefined; + try readResponse( + &reader, + .partial_content, + .{ .start = 2, .end = 9, .total = 10 }, + 3, + &.{ &first, &second }, + 5, + null, + ); + try std.testing.expectEqualStrings("34", &first); + try std.testing.expectEqualStrings("567", &second); +} + +test "200 responses that ignore Range are positioned and scattered" { + var reader: std.Io.Reader = .fixed("0123456789"); + var first: [1]u8 = undefined; + var second: [4]u8 = undefined; + try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5, null); + try std.testing.expectEqualStrings("3", &first); + try std.testing.expectEqualStrings("4567", &second); +} + +test "partial responses require a covering Content-Range" { + var reader: std.Io.Reader = .fixed("3456"); + var output: [4]u8 = undefined; + try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4, null)); + + reader = .fixed("4567"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4, null), + ); + + reader = .fixed("345"); + try std.testing.expectError( + error.InvalidContentRange, + readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4, null), + ); +} diff --git a/vfs/request.zig b/vfs/request.zig new file mode 100644 index 00000000..9365d460 --- /dev/null +++ b/vfs/request.zig @@ -0,0 +1,1212 @@ +//! One governed HTTP request loop for every backend: wait out the backend's +//! hold, send, classify. A throttle (429 everywhere, 503 on the object +//! stores) arms a backend-wide hold instead of charging the request's retry +//! budget, so every reader of that backend backs off together and the traffic +//! actually falls; every other retryable failure keeps the per-request +//! backoff. The hold is a deadline, not a permit: nothing is narrowed +//! permanently, recovery is the deadline passing, and a reader's width stays +//! the ceiling it chose, because this loop runs inside its reads. + +const std = @import("std"); + +const log = @import("base.zig").scopedLog(.@"zml/vfs/request"); + +/// The retry knobs of one remote backend: its `Options.retry`. +pub const Retry = struct { + /// Retries per request for failures that are not rate limiting. + max_retries: usize = 5, + initial_delay: std.Io.Duration = .fromMilliseconds(500), + /// Longest backoff before a request's next attempt after a failure that + /// is not a throttle, a server-named delay included. A throttle waits + /// out the backend's hold instead (`max_hold`). + max_delay: std.Io.Duration = .fromSeconds(30), + /// Longest hold one throttle may arm over every request of the backend, + /// a server-named delay included. + max_hold: std.Io.Duration = .fromSeconds(120), + /// A throttling episode with no clean window longer than this fails the + /// backend's requests. + throttle_budget: std.Io.Duration = .fromSeconds(300), + /// An exchange that receives nothing for this long is canceled and + /// retried as a timeout: a peer that stops sending without closing the + /// connection would otherwise hold its reader, and the load waiting for + /// it, forever. Body readers count what they receive + /// (`Attempt.progress`); an exchange that counts nothing (a HEAD, a token + /// exchange) must finish within it. + stall_timeout: std.Io.Duration = .fromSeconds(60), +}; + +/// What a hold covers. One governor holds one scope, its backend instance; +/// the authority is carried so that a per-authority map is a local change in +/// `Governor.holdFor`. +pub const Key = []const u8; + +/// The rate-limit scope of a request: its URI authority. Only the host is +/// taken, since `Governor.holdFor` ignores the key; a per-authority scope +/// would want the port beside it. +pub fn authorityOf(uri: std.Uri) Key { + const host = uri.host orelse return ""; + return switch (host) { + .raw => |raw| raw, + .percent_encoded => |encoded| encoded, + }; +} + +/// One scope's rate-limit state, owned by the governor's mutex. +const Hold = struct { + /// When requests may go again; null when nothing is held. + until: ?std.Io.Timestamp = null, + /// The length of the current hold: a clean window that long ends the + /// episode. + length: std.Io.Duration = .fromNanoseconds(0), + /// Holds armed since the episode began; the backoff base. + consecutive: usize = 0, + episode_started: ?std.Io.Timestamp = null, + /// The episode outlived `throttle_budget`; requests fail until a clean + /// window clears it. + exhausted: bool = false, +}; + +/// One per backend instance: what each of its requests needs, its +/// rate-limit state and its read statistics. +pub const Governor = struct { + pub const Stats = struct { + physical_requests: std.atomic.Value(u64) = .init(0), + physical_bytes: std.atomic.Value(u64) = .init(0), + retries: std.atomic.Value(u64) = .init(0), + transient_retries: std.atomic.Value(u64) = .init(0), + timeouts: std.atomic.Value(u64) = .init(0), + server_failures: std.atomic.Value(u64) = .init(0), + throttles: std.atomic.Value(u64) = .init(0), + retry_delay_ns: std.atomic.Value(u64) = .init(0), + /// Backend-wide holds armed by rate limiting, and the time requests + /// spent waiting them out. + holds: std.atomic.Value(u64) = .init(0), + hold_wait_ns: std.atomic.Value(u64) = .init(0), + + fn recordAttempt(self: *Stats) void { + _ = self.physical_requests.fetchAdd(1, .monotonic); + } + + pub fn recordSuccess(self: *Stats, request_size: usize) void { + _ = self.physical_bytes.fetchAdd(@intCast(request_size), .monotonic); + } + + fn recordFailure(self: *Stats, failure: ReadFailure) void { + switch (failure) { + .transient => _ = self.transient_retries.fetchAdd(1, .monotonic), + .timeout => _ = self.timeouts.fetchAdd(1, .monotonic), + .server_failure => _ = self.server_failures.fetchAdd(1, .monotonic), + .throttle => _ = self.throttles.fetchAdd(1, .monotonic), + } + } + + fn recordRetry(self: *Stats) void { + _ = self.retries.fetchAdd(1, .monotonic); + } + + fn recordRetryDelay(self: *Stats, delay: std.Io.Duration) void { + const delay_ns: u64 = @intCast(@max(delay.nanoseconds, 0)); + _ = self.retry_delay_ns.fetchAdd(delay_ns, .monotonic); + } + + /// One rate-limit hold armed over every request of this backend. + fn recordHold(self: *Stats) void { + _ = self.holds.fetchAdd(1, .monotonic); + } + + /// Time one request spent waiting a hold out. + fn recordHoldWait(self: *Stats, waited: std.Io.Duration) void { + _ = self.hold_wait_ns.fetchAdd(@intCast(@max(waited.nanoseconds, 0)), .monotonic); + } + }; + + io: std.Io, + client: *std.http.Client, + /// The backend's name, for log lines. + name: []const u8, + /// What a 503 means for this backend: rate limiting on the object stores + /// (AWS `SlowDown`, GCS), a server failure elsewhere. + unavailable: ReadFailure, + retry: Retry, + stats: Stats = .{}, + mutex: std.Io.Mutex = .init, + hold: Hold = .{}, + + pub fn init(io: std.Io, client: *std.http.Client, name: []const u8, unavailable: ReadFailure, retry: Retry) Governor { + std.debug.assert(retry.initial_delay.nanoseconds >= 0); + std.debug.assert(retry.max_delay.nanoseconds >= retry.initial_delay.nanoseconds); + std.debug.assert(retry.max_hold.nanoseconds >= retry.initial_delay.nanoseconds); + return .{ .io = io, .client = client, .name = name, .unavailable = unavailable, .retry = retry }; + } + + /// Logs the read totals at debug, once, from the backend's `deinit`; + /// nothing when the backend sent no request. Holds, retries and failures + /// are logged at warn as they happen. + pub fn logTotals(self: *const Governor) void { + const stats = &self.stats; + const requests = stats.physical_requests.load(.acquire); + if (requests == 0) return; + // Under the VFS's scope, `zml/vfs`. + std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, holds={d}, hold_wait_ms={d:.3}, retry_delay_ms={d:.3}", .{ + self.name, + requests, + stats.physical_bytes.load(.acquire), + stats.retries.load(.acquire), + stats.transient_retries.load(.acquire), + stats.timeouts.load(.acquire), + stats.server_failures.load(.acquire), + stats.throttles.load(.acquire), + stats.holds.load(.acquire), + @as(f64, @floatFromInt(stats.hold_wait_ns.load(.acquire))) / std.time.ns_per_ms, + @as(f64, @floatFromInt(stats.retry_delay_ns.load(.acquire))) / std.time.ns_per_ms, + }); + } + + /// A failed open, stat or access of `path` as `OpenError` reports it. + pub fn openFailure(self: *const Governor, path: []const u8, err: anyerror) OpenError { + return switch (err) { + error.FileNotFound => error.FileNotFound, + error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + error.OutOfMemory => error.SystemResources, + else => { + log.err("{s}: looking up {s} failed: {t}", .{ self.name, path, err }); + return error.Unexpected; + }, + }; + } + + /// A failed read of `target` at `offset` as a backend's read reports it: + /// a canceled task stays `Canceled`, since every wait in the governed + /// loop is a cancellation point; any other failure is logged once at + /// err, by `exchange` or here, and is `Unexpected`. + pub fn readFailure(self: *const Governor, target: []const u8, offset: u64, err: anyerror) error{ Canceled, Unexpected } { + return switch (err) { + error.Canceled => error.Canceled, + // Logged by `exchange`. + error.RequestFailed => error.Unexpected, + else => { + log.err("{s}: reading {s} at {d} failed: {t}", .{ self.name, target, offset, err }); + return error.Unexpected; + }, + }; + } + + /// The scope `key` belongs to. One generic `HTTP` instance serves every + /// host of its scheme and one `S3` spans buckets, so per-instance + /// over-reaches there; a per-authority scope replaces this body with a + /// map without touching the loop. + fn holdFor(self: *Governor, key: Key) *Hold { + _ = key; + return &self.hold; + } +}; + +/// One attempt as a backend's request hook sees it. +pub const Attempt = struct { + /// The backend and the request as `perform` was given them, which + /// `exchange` sends and logs with. + governor: *Governor, + spec: RequestSpec, + /// The status the previous attempt returned, for a hook that reacts to + /// one (an expired signed URL answering 403 after a long hold). + previous_status: ?std.http.Status = null, + /// Bytes received so far, for a hook whose body is long: `exchange` + /// cancels an exchange this counter does not move for + /// `Retry.stall_timeout`. + progress: *std.atomic.Value(u64), +}; + +/// How often a watched exchange's progress is checked, per stall timeout. +const stall_checks = 12; + +/// Identity of one request, for the logs and for the classification. +pub const RequestSpec = struct { + /// Object identity for log lines. + target: []const u8, + /// The rate-limit scope: the request URI's authority. + key: Key = "", + /// Statuses this request retries although they are not retryable in + /// general, because its hook can do something about them: a signed + /// download URL that expired during a hold answers 401 or 403, and the + /// hook re-resolves it. Charged to the retry budget. + retry_once: []const std.http.Status = &.{}, +}; + +pub const ReadFailure = enum { + transient, + timeout, + server_failure, + throttle, +}; + +pub const Failure = struct { + failure: ReadFailure, + /// The delay the server named, if any. + delay: ?std.Io.Duration = null, + status: ?std.http.Status = null, +}; + +/// What one attempt produced: a value, or a reason to try again. +pub fn Outcome(comptime T: type) type { + return union(enum) { + done: T, + retry: Failure, + }; +} + +/// The one request loop. `attemptFn` performs a single HTTP exchange (see +/// `exchange`); this drives the hold, the classification and the two budgets +/// around it. A throttle re-runs the attempt without charging a retry, after +/// the hold it armed; any other retryable failure, a stall +/// (`Retry.stall_timeout`) included, sleeps the per-request backoff and charges +/// one of `max_retries`. A cancel of the caller propagates. +pub fn perform( + comptime T: type, + governor: *Governor, + spec: RequestSpec, + context: anytype, + comptime attemptFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), +) anyerror!T { + var charged: usize = 0; + var previous_status: ?std.http.Status = null; + while (true) { + try admit(governor, spec); + governor.stats.recordAttempt(); + var progress: std.atomic.Value(u64) = .init(0); + switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status, .progress = &progress })) { + .done => |value| return value, + .retry => |failure| { + governor.stats.recordFailure(failure.failure); + previous_status = failure.status; + if (failure.failure == .throttle) { + // Not charged to the retry budget: the hold is what + // reduces the traffic, and the episode budget bounds it. + try reportThrottle(governor, spec, failure.delay); + continue; + } + if (charged >= governor.retry.max_retries) return error.RetriesExhausted; + charged += 1; + governor.stats.recordRetry(); + const retry = governor.retry; + const delay = if (failure.delay) |named| + clampDelay(named, .fromNanoseconds(0), retry.max_delay) + else + fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, charged); + governor.stats.recordRetryDelay(delay); + try governor.io.sleep(delay, .awake); + }, + } + } +} + +/// Runs one exchange on its own unit of concurrency while this task watches +/// the attempt's progress, and cancels it after `Retry.stall_timeout` without +/// progress. The exchange returns a canceled read as `Canceled`, unlogged, +/// so the outcome of a stalled exchange is decided here, as a timeout. +/// Without a unit of concurrency the exchange runs unwatched. +fn watched( + comptime T: type, + context: anytype, + comptime exchangeFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), + attempt: Attempt, +) anyerror!Outcome(T) { + const Run = struct { + result: anyerror!Outcome(T) = undefined, + done: std.Io.Event = .unset, + + fn run(self: *@This(), io: std.Io, context_: @TypeOf(context), attempt_: Attempt) void { + self.result = exchangeFn(context_, attempt_); + self.done.set(io); + } + }; + const governor = attempt.governor; + var run: Run = .{}; + var future = governor.io.concurrent(Run.run, .{ &run, governor.io, context, attempt }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return exchangeFn(context, attempt), + }; + const stall_timeout = governor.retry.stall_timeout; + const check: std.Io.Timeout = .{ .duration = .{ + .raw = .fromNanoseconds(@divTrunc(stall_timeout.nanoseconds, stall_checks)), + .clock = .awake, + } }; + var seen = attempt.progress.load(.monotonic); + var quiet: usize = 0; + while (quiet < stall_checks) { + run.done.waitTimeout(governor.io, check) catch |err| switch (err) { + error.Timeout => { + const now = attempt.progress.load(.monotonic); + quiet = if (now == seen) quiet + 1 else 0; + seen = now; + continue; + }, + error.Canceled => { + future.cancel(governor.io); + // An exchange already past its last cancelation point + // finished: its value is the caller's, and the cancel + // reaches the caller at its next cancelation point. + if (run.result) |outcome| switch (outcome) { + .done => { + governor.io.recancel(); + return outcome; + }, + .retry => {}, + } else |_| {} + return error.Canceled; + }, + }; + future.await(governor.io); + return run.result; + } + future.cancel(governor.io); + // It may have finished while being canceled. + if (run.result) |outcome| switch (outcome) { + .done => return outcome, + .retry => {}, + } else |_| {} + log.warn("{s}: {s} received nothing for {d:.1}s; retrying", .{ governor.name, attempt.spec.target, @as(f64, @floatFromInt(stall_timeout.nanoseconds)) / std.time.ns_per_s }); + return .{ .retry = .{ .failure = .timeout } }; +} + +/// Waits out the scope's hold. Returns at once when nothing is held; fails +/// with `error.RateLimited` when the episode outlived its budget. Waiters +/// wake at the deadline plus their own jitter, so the readers of one backend +/// do not all fire in the same instant, and re-check, since the hold may have +/// been extended while they slept. +fn admit(governor: *Governor, spec: RequestSpec) error{ Canceled, RateLimited }!void { + while (true) { + const deadline = deadline: { + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + if (hold.exhausted) return error.RateLimited; + const until = hold.until orelse return; + if (until.nanoseconds <= now.nanoseconds) return; + break :deadline until; + }; + const jitter = fullJitterDelay(governor.io, governor.retry.initial_delay, governor.retry.initial_delay, 0); + const started: std.Io.Timestamp = .now(governor.io, .awake); + try std.Io.Clock.Timestamp.wait(deadline.addDuration(jitter).withClock(.awake), governor.io); + governor.stats.recordHoldWait(started.durationTo(.now(governor.io, .awake))); + } +} + +/// Arms or extends the scope's hold after a throttle. `named` is the delay +/// the server asked for, if any; it is honoured up to `max_hold` and never +/// below `initial_delay`, because `Retry-After: 0` and Hugging Face's +/// `RateLimit; t=0` parse to zero, and a throttle charges no retry: a zero +/// hold would be a hot loop. +fn reportThrottle(governor: *Governor, spec: RequestSpec, named: ?std.Io.Duration) error{RateLimited}!void { + const retry = governor.retry; + governor.mutex.lockUncancelable(governor.io); + defer governor.mutex.unlock(governor.io); + const hold = governor.holdFor(spec.key); + const now: std.Io.Timestamp = .now(governor.io, .awake); + refresh(hold, now); + + if (hold.episode_started) |started| { + if (started.durationTo(now).nanoseconds > retry.throttle_budget.nanoseconds) { + hold.exhausted = true; + // The caller reports the failure; this is the one line that + // says why its request never ran. + log.warn("{s}: rate limited for longer than the throttle budget while reading {s}; giving up", .{ governor.name, spec.target }); + return error.RateLimited; + } + } else { + hold.episode_started = now; + } + + const holding = if (hold.until) |until| now.nanoseconds < until.nanoseconds else false; + if (!holding) { + hold.consecutive += 1; + const backoff = fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, hold.consecutive - 1); + var delay = backoff; + if (named) |value| { + if (value.nanoseconds > delay.nanoseconds) delay = value; + } + hold.length = clampDelay(delay, retry.initial_delay, retry.max_hold); + hold.until = now.addDuration(hold.length); + log.warn("{s}: {s} is rate limiting; holding its requests for {d:.3}s", .{ + governor.name, + spec.target, + @as(f64, @floatFromInt(hold.length.toNanoseconds())) / std.time.ns_per_s, + }); + governor.stats.recordHold(); + return; + } + // A request already in flight when the hold armed got its own answer, + // with a later reset: extend, never shorten. + const asked = named orelse return; + const capped = clampDelay(asked, retry.initial_delay, retry.max_hold); + const extended = now.addDuration(capped); + if (extended.nanoseconds > hold.until.?.nanoseconds) { + hold.until = extended; + hold.length = capped; + } +} + +/// A clean window as long as the hold itself, with no new throttle, ends the +/// episode: nothing else needs counting, since a throttle inside the window +/// would have re-armed `until`. +fn refresh(hold: *Hold, now: std.Io.Timestamp) void { + const until = hold.until orelse return; + if (now.nanoseconds < until.addDuration(hold.length).nanoseconds) return; + hold.* = .{}; +} + +fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { + return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); +} + +/// The hops `Redirects.follow` takes: std's default. +pub const max_redirects = 3; + +/// What happens to a redirect: the client chases it (`max_redirects` hops), +/// `consume` receives it because `HeadChain` walks the chain itself, or it +/// fails the request. +pub const Redirects = enum { + follow, + surface, + forbid, + + fn behavior(self: Redirects) std.http.Client.Request.RedirectBehavior { + return switch (self) { + .follow => .init(max_redirects), + .surface => .unhandled, + .forbid => .not_allowed, + }; + } +}; + +pub const ExchangeOptions = struct { + method: std.http.Method = .GET, + authorization: std.http.Client.Request.Headers.Value = .default, + content_type: std.http.Client.Request.Headers.Value = .default, + extra_headers: []const std.http.Header = &.{}, + /// Sent as the whole request body; the method must allow one. The + /// writer uses it as its own buffer, so it must be mutable. + payload: ?[]u8 = null, + /// An open, stat or access: an absent or denied object fails the + /// request with its `OpenError` before `consume`, unlogged. + open: bool = false, + redirects: Redirects = .forbid, + /// A refusal that fails the request is logged with up to 1 KiB of its + /// body, where an OAuth endpoint says why, instead of with its head. + log_refusal_body: bool = false, +}; + +/// One HTTP exchange of `attempt`'s request: send, receive the head, +/// classify it, and hand a 2xx (or a redirect `Redirects.surface` hands +/// back) to `consume` while the request is alive. Retryable connection, +/// send and receive failures and every classified status become `.retry`; +/// an unclassified status or any other connect or head failure fails with +/// `error.RequestFailed`, logged at err, except that an open request +/// (`ExchangeOptions.open`) fails an absent or denied object with +/// `FileNotFound` or `PermissionDenied` first, unlogged. The exchange alone +/// is watched (`Retry.stall_timeout`); a governed request its hook runs +/// first (a re-resolve, a token refresh) is watched by its own exchange. +pub fn exchange( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const Exchange = struct { + uri: std.Uri, + options: ExchangeOptions, + context: @TypeOf(context), + + fn run(self: @This(), current: Attempt) anyerror!Outcome(T) { + return roundTrip(T, current, self.uri, self.options, self.context, consume); + } + }; + return watched(T, Exchange{ .uri = uri, .options = options, .context = context }, Exchange.run, attempt); +} + +fn roundTrip( + comptime T: type, + attempt: Attempt, + uri: std.Uri, + options: ExchangeOptions, + context: anytype, + comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, +) anyerror!Outcome(T) { + const spec = attempt.spec; + // A followed redirect keeps each hop's Location and merged path here, + // 8 KiB of each, as much as a `HeadChain` URL; `req.uri` points into it. + var redirect_buffer: [2 * max_redirects * 8 * 1024]u8 = undefined; + var req = attempt.governor.client.request(options.method, uri, .{ + .redirect_behavior = options.redirects.behavior(), + .headers = .{ + // Range offsets and Content-Length count the stored bytes. + .accept_encoding = .{ .override = "identity" }, + .authorization = options.authorization, + .content_type = options.content_type, + }, + .extra_headers = options.extra_headers, + }) catch |err| switch (err) { + error.Timeout => return retryable(T, attempt, "connect", err, .timeout), + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "connect", err, .transient), + // A stalled exchange that `watched` canceled, or the caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "connect", err), + }; + defer req.deinit(); + + // A request that was not sent whole leaves its connection unusable; + // `deinit` would otherwise pool it, its reader still `ready`. + if (options.payload) |payload| { + req.sendBodyComplete(payload) catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send body", err, .transient); + }, + }; + } else { + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + req.connection.?.closing = true; + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "send headers", err, .transient); + }, + }; + } + + var res = req.receiveHead(if (options.redirects == .follow) &redirect_buffer else &.{}) catch |err| { + // A failed read already closed it; a redirect re-sends on a new + // connection that a failure leaves unusable too (or null). + if (req.connection) |connection| connection.closing = true; + if (canceled(&req)) return error.Canceled; + switch (err) { + error.Timeout => return retryable(T, attempt, "receive headers", err, .timeout), + error.HttpConnectionClosing, + error.HttpRequestTruncated, + error.ReadFailed, + error.WriteFailed, + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.NameServerFailure, + => return retryable(T, attempt, "receive headers", err, .transient), + // A stalled exchange that `watched` canceled, or the caller's cancel. + error.Canceled => return err, + error.OutOfMemory => return err, + else => return fatal(attempt, "receive headers", err), + } + }; + + if (options.open) { + if (openStatus(res.head.status)) |err| return err; + } + const class = res.head.status.class(); + if (class != .success and !(class == .redirect and options.redirects == .surface)) { + for (spec.retry_once) |status| { + if (status != res.head.status) continue; + log.warn("{s}: {s} answered {d}; retrying once with a fresh request", .{ + attempt.governor.name, + spec.target, + @backingInt(res.head.status), + }); + return .{ .retry = .{ .failure = .transient, .status = res.head.status } }; + } + const failure = classifyStatus(res.head.status, attempt.governor.unavailable) orelse { + if (!options.log_refusal_body) { + log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return error.RequestFailed; + } + // Reading the body invalidates the head's strings. + const status = res.head.status; + var body_buffer: [1024]u8 = undefined; + var body: std.Io.Writer = .fixed(&body_buffer); + // A longer body is cut, one that fails is logged as it arrived. + _ = res.reader(&.{}).streamRemaining(&body) catch { + if (canceled(&req)) return error.Canceled; + }; + // One line: an OAuth endpoint spreads its JSON over several. + std.mem.replaceScalar(u8, body.buffered(), '\n', ' '); + log.err("{s}: {s} answered {d}: {s}", .{ attempt.governor.name, spec.target, @backingInt(status), body.buffered() }); + return error.RequestFailed; + }; + log.warn("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); + return .{ .retry = .{ + .failure = failure, + .delay = serverRetryDelay(res.head), + .status = res.head.status, + } }; + } + + return .{ .done = consume(context, &res) catch |err| switch (err) { + error.EndOfStream, error.ReadFailed => { + if (canceled(&req)) return error.Canceled; + return retryable(T, attempt, "read body", err, .transient); + }, + else => return err, + } }; +} + +/// Whether the exchange's read or write failed because it was canceled: +/// std's stream keeps the cancel and reports a failed read or write. +fn canceled(req: *const std.http.Client.Request) bool { + const connection = req.connection orelse return false; + if (connection.stream_reader.err) |err| return err == error.Canceled; + if (connection.stream_writer.err) |err| return err == error.Canceled; + return false; +} + +fn retryable( + comptime T: type, + attempt: Attempt, + stage: []const u8, + err: anyerror, + failure: ReadFailure, +) Outcome(T) { + log.warn("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return .{ .retry = .{ .failure = failure } }; +} + +fn fatal(attempt: Attempt, stage: []const u8, err: anyerror) error{RequestFailed} { + log.err("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); + return error.RequestFailed; +} + +/// The URLs of a HEAD redirect chain walked by hand: std hands a HEAD its +/// redirect unfollowed. Three buffers in rotation: the current URL, the +/// `Location` it answered (copied out of a head that dies with the +/// response) with the path merged against it, and the resolved next URL. +pub const HeadChain = struct { + urls: [3][8 * 1024]u8 = undefined, + + pub const End = struct { + /// `start_url` itself, or a URL in the chain's buffers. + url: []const u8, + /// Its `Content-Length`, when it sent one. + size: ?u64, + }; + + /// Follows `start_url` through at most `max_hops` redirects, absolute + /// or relative, to the URL that answers a 2xx. Each hop is one governed + /// HEAD, so a server that rate limits it holds the backend as a + /// throttled GET would. `authorize` names the credentials each URL is + /// sent. An absent or denied object fails the walk + /// (`ExchangeOptions.open`). + pub fn resolve( + self: *HeadChain, + governor: *Governor, + target: []const u8, + start_url: []const u8, + max_hops: usize, + context: anytype, + comptime authorize: fn (@TypeOf(context), url: []const u8) std.http.Client.Request.Headers.Value, + ) !End { + const Hop = struct { + context: @TypeOf(context), + url: []const u8, + uri: std.Uri, + /// Where a redirect's `Location` is copied. + location: []u8, + + const Result = union(enum) { end: ?u64, redirect: usize }; + + fn attempt(hop: *@This(), current: Attempt) anyerror!Outcome(Result) { + return exchange(Result, current, hop.uri, .{ + .method = .HEAD, + .authorization = authorize(hop.context, hop.url), + .open = true, + .redirects = .surface, + }, hop, consume); + } + + fn consume(hop: *@This(), res: *std.http.Client.Response) anyerror!Result { + if (res.head.status.class() == .success) return .{ .end = res.head.content_length }; + const location = res.head.location orelse return error.HttpRedirectLocationMissing; + if (location.len > hop.location.len) return error.HttpRedirectLocationOversize; + @memcpy(hop.location[0..location.len], location); + return .{ .redirect = location.len }; + } + }; + + var url = start_url; + // `start_url` is in none of the buffers. + var current: usize = self.urls.len - 1; + var hops: usize = 0; + while (true) { + const uri: std.Uri = try .parse(url); + const scratch = (current + 1) % self.urls.len; + var hop: Hop = .{ .context = context, .url = url, .uri = uri, .location = &self.urls[scratch] }; + const location_len = switch (try perform(Hop.Result, governor, .{ .target = target, .key = authorityOf(uri) }, &hop, Hop.attempt)) { + .end => |size| return .{ .url = url, .size = size }, + .redirect => |len| len, + }; + + if (hops == max_hops) return error.TooManyHttpRedirects; + hops += 1; + const output = (current + 2) % self.urls.len; + var aux: []u8 = &self.urls[scratch]; + const resolved = try uri.resolveInPlace(location_len, &aux); + url = try std.fmt.bufPrint(&self.urls[output], "{f}", .{resolved}); + current = output; + } + } +}; + +/// Retry classification of a non-2xx status; null when the status is not +/// retried. `unavailable` is what a 503 means for the backend: rate limiting +/// on the object stores (AWS `SlowDown`, GCS), a server failure elsewhere. +fn classifyStatus(status: std.http.Status, unavailable: ReadFailure) ?ReadFailure { + return switch (status) { + .request_timeout => .timeout, + .too_many_requests => .throttle, + .service_unavailable => unavailable, + else => if (status.class() == .server_error) .server_failure else null, + }; +} + +/// What an open, stat or access of a remote path fails with (vfs.zig's +/// header); `openStatus` maps the statuses, and anything else is logged +/// once, by `exchange` or `Governor.openFailure`. +pub const OpenError = error{ FileNotFound, PermissionDenied, Canceled, SystemResources, Unexpected }; + +/// The error a status means for the path an open request +/// (`ExchangeOptions.open`) asked about; null when the status is not one. +fn openStatus(status: std.http.Status) ?error{ FileNotFound, PermissionDenied } { + return switch (status) { + .not_found, .gone => error.FileNotFound, + .unauthorized, .forbidden => error.PermissionDenied, + else => null, + }; +} + +/// The retry delay the server names, if any: `Retry-After` delta-seconds +/// (the HTTP-date form is not parsed and falls back to the jittered delay) +/// or the `t=` reset of the `RateLimit` header Hugging Face sends. +fn serverRetryDelay(head: std.http.Client.Response.Head) ?std.Io.Duration { + var it = head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Retry-After")) { + return delaySeconds(header.value) orelse continue; + } + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const trimmed = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, trimmed, "t=")) return delaySeconds(trimmed[2..]) orelse continue; + } + } + } + return null; +} + +fn delaySeconds(value: []const u8) ?std.Io.Duration { + const seconds = std.fmt.parseInt(u32, std.mem.trim(u8, value, " \t"), 10) catch return null; + return .fromSeconds(seconds); +} + +fn fullJitterDelay( + io: std.Io, + initial: std.Io.Duration, + maximum: std.Io.Duration, + attempt: usize, +) std.Io.Duration { + const max_delay_ns: i96 = @min( + maximum.toNanoseconds(), + initial.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + if (max_delay_ns <= 0) return .fromNanoseconds(0); + + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + var prng: std.Random.DefaultPrng = .init(seed); + return .fromNanoseconds(prng.random().intRangeAtMost(i96, 0, max_delay_ns)); +} + +const test_spec: RequestSpec = .{ .target = "object" }; + +fn testGovernor(initial_ms: i64, max_delay_ms: i64, max_hold_ms: i64, budget_ms: i64) Governor { + // No attempt in these tests reaches the network: no client. + return .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(initial_ms), + .max_delay = .fromMilliseconds(max_delay_ms), + .max_hold = .fromMilliseconds(max_hold_ms), + .throttle_budget = .fromMilliseconds(budget_ms), + }); +} + +test "a throttle arms a hold of at least the initial delay, even when the server names zero" { + var governor = testGovernor(20, 20, 1000, 300_000); + + // `Retry-After: 0` and Hugging Face's `RateLimit; t=0` parse to zero. + try reportThrottle(&governor, test_spec, .fromSeconds(0)); + const hold = governor.hold; + try std.testing.expect(hold.until != null); + try std.testing.expectEqual(@as(i96, 20 * std.time.ns_per_ms), hold.length.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), hold.consecutive); + try std.testing.expect(hold.episode_started != null); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a longer named delay extends a running hold and a shorter one does not" { + var governor = testGovernor(10, 10, 1000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const armed = governor.hold.until.?; + + // A request that was already in flight got its own answer, with a later + // reset: the hold extends. + try reportThrottle(&governor, test_spec, .fromMilliseconds(500)); + const extended = governor.hold.until.?; + try std.testing.expect(extended.nanoseconds > armed.nanoseconds); + + // A shorter one never shortens it, and neither arms a second hold. + try reportThrottle(&governor, test_spec, .fromMilliseconds(1)); + try std.testing.expectEqual(extended.nanoseconds, governor.hold.until.?.nanoseconds); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); +} + +test "a named delay is capped by max_hold" { + var governor = testGovernor(10, 10, 50, 300_000); + + try reportThrottle(&governor, test_spec, .fromSeconds(3600)); + try std.testing.expectEqual(@as(i96, 50 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "a second throttle inside the clean window doubles the backoff base" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); + + // The hold has passed but the clean window has not: rewind it by hand + // rather than sleeping through a race. + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(.fromMilliseconds(1)); + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 2), governor.hold.consecutive); + // Full jitter over twice the base, floored at the initial delay. + try std.testing.expect(governor.hold.length.nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.hold.length.nanoseconds <= 20 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 2), governor.stats.holds.load(.acquire)); +} + +test "a clean window as long as the hold ends the episode" { + var governor = testGovernor(10, 1000, 10_000, 300_000); + + try reportThrottle(&governor, test_spec, null); + const length = governor.hold.length; + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.until = now.subDuration(length).subDuration(.fromMilliseconds(1)); + + try reportThrottle(&governor, test_spec, null); + try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); + try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); +} + +test "an episode longer than the throttle budget fails requests until a clean window" { + var governor = testGovernor(10, 10, 10_000, 50); + + try reportThrottle(&governor, test_spec, .fromSeconds(30)); + const now: std.Io.Timestamp = .now(std.testing.io, .awake); + governor.hold.episode_started = now.subDuration(.fromMilliseconds(51)); + + try std.testing.expectError(error.RateLimited, reportThrottle(&governor, test_spec, null)); + try std.testing.expect(governor.hold.exhausted); + // Every other request of this backend fails at once, without waiting the + // hold out. + try std.testing.expectError(error.RateLimited, admit(&governor, test_spec)); + + // The next clean window clears the episode. + governor.hold.until = now.subDuration(governor.hold.length).subDuration(.fromMilliseconds(1)); + try admit(&governor, test_spec); + try std.testing.expect(governor.hold.until == null); + try std.testing.expect(!governor.hold.exhausted); +} + +test "admit returns at once when nothing is held" { + var governor = testGovernor(10, 10, 1000, 300_000); + + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds < 5 * std.time.ns_per_ms); + try std.testing.expectEqual(@as(u64, 0), governor.stats.hold_wait_ns.load(.acquire)); +} + +test "admit waits a hold out and counts the wait" { + var governor = testGovernor(1, 1, 1000, 300_000); + + try reportThrottle(&governor, test_spec, .fromMilliseconds(15)); + const started: std.Io.Timestamp = .now(std.testing.io, .awake); + try admit(&governor, test_spec); + try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds >= 10 * std.time.ns_per_ms); + try std.testing.expect(governor.stats.hold_wait_ns.load(.acquire) > 0); +} + +test "a waiter blocked on a hold is cancelable" { + const io = std.testing.io; + var governor = testGovernor(1, 1, 100_000, 300_000); + try reportThrottle(&governor, test_spec, .fromSeconds(60)); + + var result: std.atomic.Value(u16) = .init(0); + var started: std.Io.Event = .unset; + var group: std.Io.Group = .init; + try group.concurrent(io, struct { + fn run(governor_: *Governor, started_: *std.Io.Event, result_: *std.atomic.Value(u16)) void { + started_.set(governor_.io); + admit(governor_, test_spec) catch |err| { + result_.store(@intFromError(err), .release); + return; + }; + } + }.run, .{ &governor, &started, &result }); + try started.wait(io); + try io.sleep(.fromMilliseconds(5), .awake); + group.cancel(io); + try std.testing.expectEqual(@intFromError(error.Canceled), result.load(.acquire)); +} + +test "only rate limiting holds: a timeout and a server failure keep the per-request budget" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return switch (self.seen) { + 1 => .{ .retry = .{ .failure = .timeout } }, + 2 => .{ .retry = .{ .failure = .server_failure } }, + else => .{ .done = self.seen }, + }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 3), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expect(governor.hold.until == null); + try std.testing.expectEqual(@as(u64, 0), governor.stats.holds.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 3), governor.stats.physical_requests.load(.acquire)); +} + +test "a throttle retries without charging the retry budget" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The retry budget is three; ten throttles do not exhaust it. + if (self.seen <= 10) return .{ .retry = .{ .failure = .throttle, .status = .too_many_requests } }; + try std.testing.expectEqual(@as(?std.http.Status, .too_many_requests), current.previous_status); + return .{ .done = self.seen }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectEqual(@as(usize, 11), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, 10), governor.stats.throttles.load(.acquire)); + try std.testing.expectEqual(@as(u64, 11), governor.stats.physical_requests.load(.acquire)); + try std.testing.expect(governor.stats.holds.load(.acquire) >= 1); +} + +test "the retry budget still bounds failures that are not rate limiting" { + var governor = testGovernor(0, 0, 1000, 300_000); + + const Attempts = struct { + seen: usize = 0, + fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + return .{ .retry = .{ .failure = .transient } }; + } + }; + var attempts: Attempts = .{}; + try std.testing.expectError(error.RetriesExhausted, perform(usize, &governor, test_spec, &attempts, Attempts.run)); + // The first attempt plus `max_retries` of them. + try std.testing.expectEqual(@as(usize, 4), attempts.seen); +} + +test "retry status classification is typed and 503 depends on the backend" { + try std.testing.expectEqual(ReadFailure.timeout, classifyStatus(.request_timeout, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.too_many_requests, .server_failure).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.bad_gateway, .throttle).?); + try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.service_unavailable, .server_failure).?); + try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.service_unavailable, .throttle).?); + try std.testing.expect(classifyStatus(.not_found, .throttle) == null); +} + +test "absent and denied statuses map to one error each" { + try std.testing.expect(openStatus(.not_found).? == error.FileNotFound); + try std.testing.expect(openStatus(.gone).? == error.FileNotFound); + try std.testing.expect(openStatus(.unauthorized).? == error.PermissionDenied); + try std.testing.expect(openStatus(.forbidden).? == error.PermissionDenied); + for ([_]std.http.Status{ .ok, .partial_content, .bad_request, .too_many_requests, .internal_server_error, .service_unavailable }) |status| { + try std.testing.expect(openStatus(status) == null); + } + + const governor = testGovernor(10, 10, 1000, 300_000); + try std.testing.expect(governor.openFailure("object", error.FileNotFound) == error.FileNotFound); + try std.testing.expect(governor.openFailure("object", error.PermissionDenied) == error.PermissionDenied); + try std.testing.expect(governor.openFailure("object", error.Canceled) == error.Canceled); + try std.testing.expect(governor.openFailure("object", error.OutOfMemory) == error.SystemResources); + try std.testing.expect(governor.openFailure("object", error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.openFailure("object", error.InvalidContentRange) == error.Unexpected); +} + +test "a failed read is Canceled or Unexpected" { + const governor = testGovernor(10, 10, 1000, 300_000); + try std.testing.expect(governor.readFailure("object", 0, error.Canceled) == error.Canceled); + try std.testing.expect(governor.readFailure("object", 0, error.RequestFailed) == error.Unexpected); + try std.testing.expect(governor.readFailure("object", 0, error.InvalidContentRange) == error.Unexpected); +} + +test "server retry delay comes from Retry-After seconds or a RateLimit reset" { + const Head = std.http.Client.Response.Head; + const retry_after = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nretry-after: 3\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(3), serverRetryDelay(retry_after).?); + + const rate_limit = try Head.parse("HTTP/1.1 429 Too Many Requests\r\nRateLimit: \"default\"; r=0; t=7\r\n\r\n"); + try std.testing.expectEqual(std.Io.Duration.fromSeconds(7), serverRetryDelay(rate_limit).?); + + const http_date = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nRetry-After: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"); + try std.testing.expect(serverRetryDelay(http_date) == null); + const none = try Head.parse("HTTP/1.1 500 Internal Server Error\r\n\r\n"); + try std.testing.expect(serverRetryDelay(none) == null); +} + +test "a server-named retry delay sleeps at most max_delay" { + var governor = testGovernor(1, 20, 20, 1000); + const Failing = struct { + seen: usize = 0, + fn attempt(self: *@This(), _: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // A failing server that asks for an hour. + if (self.seen == 1) return .{ .retry = .{ .failure = .server_failure, .delay = .fromSeconds(3600) } }; + return .{ .done = self.seen }; + } + }; + var failing: Failing = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &failing, Failing.attempt)); + try std.testing.expectEqual(@as(u64, 20 * std.time.ns_per_ms), governor.stats.retry_delay_ns.load(.acquire)); +} + +test "an attempt that receives nothing is canceled and retried as a timeout" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Hang = struct { + seen: usize = 0, + progressed: u64 = 0, + // Watched as `exchange` watches its round trip. + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + return watched(usize, self, receive, current); + } + + fn receive(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + self.seen += 1; + // The first exchange hangs, the second one keeps receiving and + // takes longer than the stall timeout overall. + if (self.seen == 1) { + try std.testing.io.sleep(.fromSeconds(60), .awake); + return error.TestUnexpectedResult; + } + for (0..6) |_| { + try std.testing.io.sleep(.fromMilliseconds(40), .awake); + _ = current.progress.fetchAdd(1, .monotonic); + self.progressed += 1; + } + return .{ .done = self.seen }; + } + }; + var hang: Hang = .{}; + try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &hang, Hang.attempt)); + try std.testing.expectEqual(@as(u64, 6), hang.progressed); + try std.testing.expectEqual(@as(u64, 1), governor.stats.timeouts.load(.acquire)); +} + +test "an attempt waiting on a request of its own is not a stall" { + var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ + .max_retries = 3, + .initial_delay = .fromMilliseconds(1), + .max_delay = .fromMilliseconds(1), + .max_hold = .fromMilliseconds(1), + .throttle_budget = .fromSeconds(1), + .stall_timeout = .fromMilliseconds(120), + }); + const Nested = struct { + fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { + _ = try perform(usize, current.governor, test_spec, self, wait); + return .{ .done = 1 }; + } + + // A re-resolve or token refresh waiting out a hold, outside any + // exchange, for longer than the stall timeout. + fn wait(_: *@This(), _: Attempt) anyerror!Outcome(usize) { + try std.testing.io.sleep(.fromMilliseconds(300), .awake); + return .{ .done = 0 }; + } + }; + var nested: Nested = .{}; + try std.testing.expectEqual(@as(usize, 1), try perform(usize, &governor, test_spec, &nested, Nested.attempt)); + try std.testing.expectEqual(@as(u64, 0), governor.stats.timeouts.load(.acquire)); + try std.testing.expectEqual(@as(u64, 2), governor.stats.physical_requests.load(.acquire)); +} + +test "an exchange canceled while it waits for the head returns Canceled, not a retry" { + const MockServer = @import("mock_server.zig").MockServer; + const io = std.testing.io; + + // The GET waits unanswered for a second one that never comes. + var server = try MockServer.init(io, "object", .{ .barrier_gets = 2 }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = std.testing.allocator, .io = io }; + defer client.deinit(); + var governor: Governor = .init(io, &client, "test", .throttle, .{}); + var url_buffer: [64]u8 = undefined; + const uri: std.Uri = try .parse(try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()})); + + const Exchange = struct { + // Unwatched, so that the cancel lands in the read rather than in `watched`. + fn run(governor_: *Governor, uri_: std.Uri, result: *anyerror!Outcome(void)) void { + var progress: std.atomic.Value(u64) = .init(0); + result.* = roundTrip(void, .{ .governor = governor_, .spec = test_spec, .progress = &progress }, uri_, .{ + .extra_headers = &.{.{ .name = "Range", .value = "bytes=0-5" }}, + }, {}, consume); + } + + fn consume(_: void, _: *std.http.Client.Response) anyerror!void {} + }; + var result: anyerror!Outcome(void) = .{ .done = {} }; + var group: std.Io.Group = .init; + try group.concurrent(io, Exchange.run, .{ &governor, uri, &result }); + try io.sleep(.fromMilliseconds(50), .awake); + group.cancel(io); + try std.testing.expectError(error.Canceled, result); + + try server.stop(io); +} + +test "the request authority is the rate-limit scope" { + const uri: std.Uri = try .parse("https://huggingface.co/api/models/x"); + try std.testing.expectEqualStrings("huggingface.co", authorityOf(uri)); +} diff --git a/vfs/s3.zig b/vfs/s3.zig index eb849e21..65132a64 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -2,8 +2,10 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); +const range_read = @import("range_read.zig"); +const request = @import("request.zig"); const VFSBase = @import("base.zig").VFSBase; +const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/s3"); @@ -141,185 +143,26 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *S3, - uri: std.Uri, - url: []const u8, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; - - const signer: AwsSigV4 = .{ - .access_key = job.batch.backend.config.access_key, - .secret_key = job.batch.backend.config.secret_key, - .region = job.batch.backend.config.region, - .service = job.batch.backend.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = signer.generateAuthHeader( - &authorization_buffer, - .GET, - job.batch.uri, - timestamp, - &.{.{ .name = "Range", .value = range_header }}, - ) catch |err| { - log.err("Failed to generate auth header: {}", .{err}); - return err; - }; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - }, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; - - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - - pub const Config = struct { + /// Without both keys, requests go unsigned. + pub const Options = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, endpoint_url: []const u8, region: []const u8, - auth_service: []const u8 = "s3", + retry: request.Retry = .{}, }; + /// The strings of the `Options` given to `init`, owned. + const Config = struct { + access_key: ?[]const u8, + secret_key: ?[]const u8, + endpoint_url: []const u8, + region: []const u8, + }; + + /// The SigV4 service every request is signed for. + const auth_service = "s3"; + const Handle = struct { pub const Type = enum { file, directory }; @@ -344,38 +187,24 @@ pub const S3 = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - client: *std.http.Client, config: Config, - read_pool: *ParallelRead.Pool, + governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub fn init( - allocator: std.mem.Allocator, - inner: std.Io, - http_client: *std.http.Client, - config: Config, - opts: InitOpts, - ) !S3 { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !S3 { return .{ .allocator = allocator, .base = .init(inner), - .client = http_client, - .read_pool = read_pool, + // S3 rate limiting is `503 SlowDown` as well as `429`. + .governor = .init(inner, http_client, "s3", .throttle, options.retry), .config = .{ - .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, - .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, - .endpoint_url = try allocator.dupe(u8, config.endpoint_url), - .region = try allocator.dupe(u8, config.region), - .auth_service = try allocator.dupe(u8, config.auth_service), + .access_key = if (options.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (options.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, options.endpoint_url), + .region = try allocator.dupe(u8, options.region), }, }; } @@ -422,13 +251,11 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }, .{}); + }); } pub fn deinit(self: *S3) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - + self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -455,7 +282,6 @@ pub const S3 = struct { if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); self.allocator.free(self.config.endpoint_url); self.allocator.free(self.config.region); - self.allocator.free(self.config.auth_service); } pub fn io(self: *S3) std.Io { @@ -483,6 +309,13 @@ pub const S3 = struct { }; } + pub fn backend(self: *S3) Backend { + return .{ + .io = self.io(), + .read_hints = .{ .high_latency = true }, + }; + } + fn openHandle(self: *S3) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -532,9 +365,9 @@ pub const S3 = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; + const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { + error.Canceled => return error.Canceled, + error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, }; if (total == 0) { @@ -580,11 +413,9 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); return .{ .inode = 0, @@ -599,18 +430,19 @@ pub const S3 = struct { }; } - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + } fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; - var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -636,7 +468,16 @@ pub const S3 = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const objects = self.listObjects(handle.uri) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.SystemResources, + // Logged by `exchange`. + error.RequestFailed => return error.Unexpected, + else => { + log.err("listing {s} failed: {t}", .{ handle.uri, err }); + return error.Unexpected; + }, + }; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -721,10 +562,7 @@ pub const S3 = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; + return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -734,7 +572,7 @@ pub const S3 = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @intCast(-relative_offset); + const abs_offset: u64 = @abs(relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -820,49 +658,37 @@ pub const S3 = struct { .fragment = null, }; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); + var listing: Listing = .{ .s3 = self, .uri = uri }; + var target_buffer: [256]u8 = undefined; + return request.perform([]u8, &self.governor, .{ + .target = std.fmt.bufPrint(&target_buffer, "{s}/{s}", .{ bucket, key_prefix }) catch bucket, + .key = request.authorityOf(uri), + }, &listing, Listing.attempt); + } - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; + /// The bucket listing, signed again on every attempt like every other + /// S3 request (`Signature`). + const Listing = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { + return request.exchange([]u8, current, self.uri, .{ + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{}), + .extra_headers = &self.signature.headers, + }, self, Listing.consume); } - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - // When we don't have a content length, put a reasonable limit - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } + fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { + const allocator = self.s3.allocator; + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + } + }; fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); @@ -905,93 +731,132 @@ pub const S3 = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; + fn fetchSize(self: *S3, path: []const u8) !u64 { var url_buf: [8 * 1024]u8 = undefined; - - const path = try self.resolvePath(dir, sub_path, &path_buffer); const url = try self.s3Url(path, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); - - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - return switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - else => blk: { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; + var head: SizeRequest = .{ .s3 = self, .uri = uri }; + return request.perform(u64, &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); } - fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; + /// The object's size: one signed HEAD (`ExchangeOptions.open`; S3's 403 + /// for a missing key: vfs.zig's header). + const SizeRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, - var url_buf: [8 * 1024]u8 = undefined; - const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .url = url, - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { + return request.exchange(u64, current, self.uri, .{ + .method = .HEAD, + .authorization = try self.signature.sign(self.s3, .HEAD, self.uri, &.{}), + .extra_headers = &self.signature.headers, + .open = true, + }, self, SizeRequest.consume); } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { + return res.head.content_length orelse error.MissingContentLength; + } + }; - return read_size; + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); + var signed: SignedRequest = .{ .s3 = self, .uri = std.Uri.parse(url) catch return error.BadPathName }; + return range_read.performRangeRead(&self.governor, .{ + .request = .{ .target = url, .key = request.authorityOf(signed.uri) }, + }, &signed, SignedRequest.prepare, data, offset, handle.size); } + + /// One SigV4-signed GET. The signature covers `x-amz-date` and `Range`, + /// so it is recomputed for every attempt. + const SignedRequest = struct { + s3: *S3, + uri: std.Uri, + signature: Signature = .{}, + + fn prepare(self: *SignedRequest, _: request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { + return .{ + .uri = self.uri, + .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{range}), + .extra_headers = &self.signature.headers, + }; + } + }; + + /// One attempt's SigV4 headers. Every request signs each attempt anew, + /// since the signature covers `x-amz-date`. The returned authorization + /// and `headers` point into this storage. + const Signature = struct { + timestamp: [16]u8 = undefined, + authorization: [512]u8 = undefined, + headers: [2]std.http.Header = undefined, + + /// The authorization over `uri` and `signed_headers` beside the two + /// headers it sets; omitted without credentials. + fn sign( + self: *Signature, + s3: *S3, + method: std.http.Method, + uri: std.Uri, + signed_headers: []const std.http.Header, + ) !std.http.Client.Request.Headers.Value { + const timestamp = try s3.getTimestamp(&self.timestamp); + const signer: AwsSigV4 = .{ + .access_key = s3.config.access_key, + .secret_key = s3.config.secret_key, + .region = s3.config.region, + .service = auth_service, + }; + const authorization = try signer.generateAuthHeader(&self.authorization, method, uri, timestamp, signed_headers); + self.headers = .{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }; + return if (authorization) |value| .{ .override = value } else .omit; + } + }; }; + +test "AwsSigV4 signs the HEAD, listing and Range GET that S3 sends" { + // AWS's example keys and date. The expected signatures are botocore's + // (S3SigV4Auth) for the same requests, with UNSIGNED-PAYLOAD as every + // request here is sent. + const signer: AwsSigV4 = .{ + .access_key = "AKIAIOSFODNN7EXAMPLE", + .secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + .region = "us-east-1", + .service = "s3", + }; + const timestamp = "20130524T000000Z"; + const credential = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, "; + var buffer: [512]u8 = undefined; + + const object = try std.Uri.parse("https://s3.amazonaws.com/examplebucket/test.txt"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d360d9545e619df15fd61e08c5b21c3e2219ee708af3b95ac439b0ee6fdf44b0", + (try signer.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})).?, + ); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f483aaef7a23c6af727aeed791410b7f43f0d696542829b6bf6edc1c7cb112c4", + (try signer.generateAuthHeader(&buffer, .GET, object, timestamp, &.{.{ .name = "Range", .value = "bytes=0-9" }})).?, + ); + const listing = try std.Uri.parse("https://s3.amazonaws.com/examplebucket?delimiter=%2F&list-type=2&max-keys=1000&prefix=dir%2F"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0a8491caa0b04a231c4ea74a87bde51ef60fbe69b496b206f4d07a4c9031188a", + (try signer.generateAuthHeader(&buffer, .GET, listing, timestamp, &.{})).?, + ); + // A custom endpoint signs its port as part of the host. + const minio = try std.Uri.parse("http://127.0.0.1:9000/models/dir/obj.bin"); + try std.testing.expectEqualStrings( + credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=84ff2d687bd88454372c3277af863c28ca047204936b346b80bf93456d0870d4", + (try signer.generateAuthHeader(&buffer, .HEAD, minio, timestamp, &.{})).?, + ); + + // Without both keys, requests go unsigned. + const anonymous: AwsSigV4 = .{ .access_key = "AKIAIOSFODNN7EXAMPLE", .secret_key = null, .region = "us-east-1", .service = "s3" }; + try std.testing.expectEqual(null, try anonymous.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})); +} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index fe626ff9..8a52e763 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,25 +1,87 @@ +//! A `std.Io` over local files and remote objects. A bare path or a +//! `file://` URI is served by the inner io; a URI such as +//! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend +//! registered for its scheme (`registerBackend`, or the four of `Remotes`). +//! Readers take `io()` and need not know which they read; one that must +//! (the loader, for read sizes and direct I/O) asks `fromIo`. +//! +//! The VFS, and every backend registered on it, must outlive each file +//! opened through it and each reader that keeps one open: a `Loader` may +//! keep its source files open until its `deinit`. +//! +//! Errors of a remote path, alike on `HTTP`, `HF`, `S3` and `GCS`: opening, +//! stat-ing or probing it (`access`) fails with `FileNotFound` for a missing +//! object (404, 410, or a path absent from an `HF` repository's listing), +//! `PermissionDenied` for a denied one (401, 403), `Canceled` for a canceled +//! task, and `SystemResources` when a request cannot allocate. Any other +//! failure of a request is logged at err with the backend and the path, and +//! surfaces as `Unexpected`. On S3 a principal without `s3:ListBucket` gets +//! 403 for a missing key too, so a probe of an optional object treats both +//! as absent. A positional read that still fails after its retries is +//! logged at err and surfaces as `Unexpected`; a streaming read as +//! `InputOutput`. A throttled backend holds its requests rather than failing +//! them, until its throttle budget runs out (`request.Retry`). A remote path +//! is read-only: creating or deleting a file or a directory there, or +//! opening a file for writing, is logged at err and fails with +//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied +//! one is `AccessDenied` there); it may be read past the page cache +//! (`useDirectIo`). Each remote backend logs its read totals at debug from +//! its `deinit`. const std = @import("std"); const stdx = @import("stdx"); -pub const File = @import("file.zig").File; pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; +/// A test-only HTTP origin for tests above this layer (mock_server.zig). +pub const MockServer = @import("mock_server.zig").MockServer; +const base_module = @import("base.zig"); +const direct_io = @import("direct_io.zig"); +pub const Backend = base_module.Backend; +pub const ReadHints = base_module.ReadHints; +const VFSBase = base_module.VFSBase; +/// See `useDirectIo`. +pub const DirectIo = direct_io.Policy; +/// What every read of a direct file must meet in offset, buffers and total +/// (`useDirectIo`); 0 on a system without direct I/O, where no file is. +pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignment else 0; + +test { + _ = @import("gcs.zig"); + _ = @import("http_acceptance_test.zig"); +} const log = std.log.scoped(.@"zml/vfs"); +/// For the refusals of writes to a remote path, which tests trigger on +/// purpose; every other err here still fails a test (`scopedLog`). +const refusal_log = base_module.scopedLog(.@"zml/vfs"); -const CWD_HANDLE: u32 = 0; +/// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and +/// friends resolve to their own descriptor; the cwd placeholder follows. +const CWD_HANDLE: u32 = 3; +/// Local files are the VFS's own: a bare path or a `file://` URI resolves +/// to the inner `Io`, never to a registered backend. +const local_scheme = "file"; const VFS = @This(); -const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; +const Handle = struct { + handle: u32, + backend_idx: ?usize, + flags: std.Io.File.Flags = .{ .nonblocking = false }, + /// A local file's read mode (`useDirectIo`). + direct: direct_io.Mode = .{}, + + fn innerFile(self: *const Handle) std.Io.File { + return .{ .handle = @intCast(self.handle), .flags = self.flags }; + } +}; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, -backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +backends: std.StringArrayHashMapUnmanaged(Backend) = .empty, handles: stdx.SegmentedList(Handle, 128) = .{}, closed_handles: std.ArrayList(u32) = .empty, @@ -29,10 +91,11 @@ pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); var handles: @FieldType(@This(), "handles") = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + std.debug.assert(handles.len == CWD_HANDLE + 1); return .{ .allocator = allocator, @@ -47,49 +110,135 @@ pub fn deinit(self: *VFS) void { self.backends.deinit(self.allocator); } -pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { +/// Serves `scheme` with `backend`, replacing any backend registered under +/// it. `scheme` is kept, not copied: it must outlive the VFS. +pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem.Allocator.Error!void { + std.debug.assert(!std.mem.eql(u8, scheme, local_scheme)); // file is the VFS's own (local_scheme) self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } -pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); +/// The remote schemes on one HTTP client: `https`, `hf`, `s3` and `gs`, each +/// configured from the environment through its backend's `auto` (plain +/// `HTTP.init` for `https`), the client honoring the proxy variables. +/// Initialized in place, since the backends point at the client; it must not +/// move afterwards. Close every file opened through these schemes before +/// `deinit` (deinitialize a `Loader` first); the VFS must not reach them +/// after it. Their errors follow this module's contract. +pub const Remotes = struct { + /// The proxies' allocations, which must outlive the client. + arena: std.heap.ArenaAllocator, + client: std.http.Client, + https: HTTP, + hf: HF, + s3: S3, + gs: GCS, - return self.backends.remove(scheme); -} + /// Builds the client on the VFS's base io and registers the four + /// schemes, replacing any backend registered under them. `environ` is + /// read here only: tokens, credentials and proxies are copied. On error + /// nothing is registered. + pub fn init(self: *Remotes, allocator: std.mem.Allocator, vfs: *VFS, environ: *std.process.Environ.Map) !void { + const base_io = vfs.base.inner; + self.arena = .init(allocator); + errdefer self.arena.deinit(); + self.client = .{ .allocator = allocator, .io = base_io }; + errdefer self.client.deinit(); + try self.client.initDefaultProxies(self.arena.allocator(), environ); + self.https = try .init(allocator, base_io, &self.client, .{ .protocol = .https }); + errdefer self.https.deinit(); + self.hf = try .auto(allocator, base_io, &self.client, environ); + errdefer self.hf.deinit(); + self.s3 = try .auto(allocator, base_io, &self.client, environ); + errdefer self.s3.deinit(); + self.gs = try .auto(allocator, base_io, &self.client, environ); + errdefer self.gs.deinit(); + vfs.mutex.lockUncancelable(base_io); + defer vfs.mutex.unlock(base_io); + // Room first: once a scheme is registered, nothing can fail. + try vfs.backends.ensureUnusedCapacity(vfs.allocator, 4); + vfs.backends.putAssumeCapacity("https", self.https.backend()); + vfs.backends.putAssumeCapacity("hf", self.hf.backend()); + vfs.backends.putAssumeCapacity("s3", self.s3.backend()); + vfs.backends.putAssumeCapacity("gs", self.gs.backend()); + } + + pub fn deinit(self: *Remotes) void { + self.gs.deinit(); + self.s3.deinit(); + self.hf.deinit(); + self.https.deinit(); + self.client.deinit(); + self.arena.deinit(); + } +}; + +/// The `std.Io` of this VFS. It points into the VFS, which must not move +/// once this was called. pub fn io(self: *VFS) std.Io { return .{ .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), + .vtable = &io_vtable, }; } +/// The VFS behind an `Io`, or null for any other implementation: a reader +/// that opened a file through `any_io` asks this VFS about that handle. +pub fn fromIo(any_io: std.Io) ?*VFS { + if (any_io.vtable != &io_vtable) return null; + return @fieldParentPtr("base", VFSBase.as(any_io.userdata)); +} + +const io_vtable: std.Io.VTable = VFSBase.vtable(.{ + .operate = operate, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirCreateFileAtomic = dirCreateFileAtomic, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, +}); + +/// The read hints of the backend serving `path`, for a reader sizing its +/// requests. A path resolves as it does for `openFile`: a bare path or a +/// `file://` URI is local (`ReadHints.local`), and a scheme no backend +/// serves is `VFSNotRegistered`. +pub fn readHints(self: *VFS, path: []const u8) !ReadHints { + const backend_idx, _, _ = try self.lookupDir(.cwd(), path); + const index = backend_idx orelse return .local; + return self.backends.entries.items(.value)[index].read_hints; +} + +/// Whether `path` resolves to a local file, the only kind `useDirectIo` can +/// read directly; an unregistered scheme is not. Opens nothing. +pub fn isLocal(self: *VFS, path: []const u8) bool { + const backend_idx, _, _ = self.lookupDir(.cwd(), path) catch return false; + return backend_idx == null; +} + fn openHandle(self: *VFS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -117,6 +266,53 @@ fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { return .{ handle, self.getBackend(handle.backend_idx) }; } +/// Direct reads for a local file under `policy`: sets the flag on a +/// read-only descriptor of a regular file at least one alignment unit long +/// (and, under `auto`, mostly out of the page cache) and returns true, after +/// which every read of the file must meet `direct_io_alignment` in offset, +/// buffers and total. False leaves the file buffered: another backend's +/// file, `off`, a cached file under `auto`, a filesystem that refuses the +/// flag, or a file whose direct read the kernel already rejected. The first +/// call under `on` or `auto` decides for the handle's life +/// (`direct_io.Mode`); the caller must ask before it reads. The mechanism +/// lives in the VFS because it owns the descriptor; one descriptor suffices +/// because a reader reads a file in one mode. +pub fn useDirectIo(self: *VFS, file: std.Io.File, policy: DirectIo) bool { + const handle, _ = self.getFileHandle(file); + if (handle.backend_idx != null) return false; + return handle.direct.decide(self.base.inner, handle.innerFile(), policy); +} + +/// See `predictDirectIo`. +pub const DirectIoPrediction = direct_io.Prediction; + +/// What `useDirectIo` would answer under `policy` for the local file +/// `sub_path` of `dir`, and its size, asked on a descriptor of its own that +/// is closed again: no handle is decided and nothing is logged. For a reader +/// that places memory before it plans its files. Under `auto` this sample +/// caches what it misses, so a decision that follows may find any file +/// cached and read it buffered, the likelier the fewer pages it has per +/// sample (`direct_io.predict`). Null for a path that is not local, or a +/// file that does not open or stat. +pub fn predictDirectIo(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, policy: DirectIo) error{Canceled}!?DirectIoPrediction { + const backend_idx, const inner_dir, const inner = self.lookupDir(dir, sub_path) catch return null; + if (backend_idx != null) return null; + const file = inner_dir.openFile(inner, stripScheme(sub_path), .{ .mode = .read_only }) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return null, + }; + defer file.close(inner); + return direct_io.predict(inner, file, policy); +} + +/// A local file as the inner `Io` may read it, off direct reads +/// (`direct_io.Mode.leave`). +fn innerReadable(self: *VFS, handle: *Handle) std.Io.File { + const file = handle.innerFile(); + handle.direct.leave(self.base.inner, file); + return file; +} + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -130,7 +326,7 @@ fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { } fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx].io else return self.base.inner; } fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { @@ -141,6 +337,7 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz if (sub_path) |sp| { if (std.mem.indexOf(u8, sp, "://") != null) { const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + if (std.mem.eql(u8, uri.scheme, local_scheme)) return .{ null, std.Io.Dir.cwd(), self.base.inner }; const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { if (std.mem.eql(u8, uri.scheme, s)) break idx; } else return error.VFSNotRegistered; @@ -149,19 +346,8 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } + // Without "://", a path from cwd is local, a colon in it included. + return .{ null, dir, self.base.inner }; } else { const handle = self.getDirHandle(dir); if (handle.backend_idx) |backend_idx| { @@ -172,9 +358,26 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } } +/// `path` without its "scheme://" prefix; a name with a colon and no "://" +/// is a plain local name. fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; + const i = std.mem.indexOf(u8, path, "://") orelse return path; + return path[i + 3 ..]; +} + +/// The inner `Io`'s directory for creating or deleting `sub_path` from +/// `dir`. A remote path is refused: no remote backend writes, and the base +/// layer under each one would write a local file in its place. +fn writableDir(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, operation: []const u8) error{ ReadOnlyFileSystem, Unexpected }!std.Io.Dir { + const backend_idx, const dir_, _ = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend to {s} '{s}' : {any}", .{ operation, sub_path, err }); + return error.Unexpected; + }; + if (self.getScheme(backend_idx)) |scheme| { + refusal_log.err("Cannot {s} '{s}': {s} paths are read-only", .{ operation, sub_path, scheme }); + return error.ReadOnlyFileSystem; + } + return dir_; } fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { @@ -183,7 +386,7 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .file_read_streaming => |o| { const handle, const backend = self.getFileHandle(o.file); return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .file = self.innerReadable(handle), .data = o.data, } }); }, @@ -202,6 +405,32 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable } } +fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDir(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + return self.base.inner.vtable.dirCreateDirPath(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); +} + +fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create directory"); + const inner = self.base.inner; + const fs_dir = try inner.vtable.dirCreateDirPathOpen(inner.userdata, dir_, stripScheme(sub_path), permissions, options); + errdefer fs_dir.close(inner); + const idx, const handle = self.openHandle() catch return std.Io.Dir.CreateDirPathOpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = null, + }; + return .{ .handle = @intCast(idx) }; +} + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -246,28 +475,36 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + const inner = self.base.inner; + const file = try inner.vtable.dirCreateFile(inner.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(inner); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), - .backend_idx = backend_idx, + .backend_idx = null, .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = file.flags }; } +/// The atomic file holds the inner `Io`'s handles, not this VFS's. +fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "create file"); + return self.base.inner.vtable.dirCreateFileAtomic(self.base.inner.userdata, dir_, stripScheme(sub_path), options); +} + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; + if (flags.mode != .read_only and backend_idx != null) { + refusal_log.err("Cannot open '{s}' for writing: {s} paths are read-only", .{ sub_path, self.getScheme(backend_idx).? }); + return error.ReadOnlyFileSystem; + } const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; @@ -330,13 +567,25 @@ fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8 if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer[prefix.len..]); return prefix.len + path_len; } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer); } } +fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete file"); + return self.base.inner.vtable.dirDeleteFile(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + +fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const dir_ = try self.writableDir(dir, sub_path, "delete directory"); + return self.base.inner.vtable.dirDeleteDir(self.base.inner.userdata, dir_, stripScheme(sub_path)); +} + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); @@ -376,7 +625,7 @@ fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []co const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -397,7 +646,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + reader.file = self.innerReadable(src_handle); defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -409,6 +658,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); + if (try handle.direct.read(self.base.inner, handle.innerFile(), data, offset)) |bytes| return bytes; return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); } @@ -437,6 +687,281 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. } } +test "VFS gives the read hints of local and registered paths" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + const hints: ReadHints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true }; + try filesystem.registerBackend("test", .{ .io = std.testing.io, .read_hints = hints }); + + try std.testing.expectEqual(hints, try filesystem.readHints("test://bucket/object")); + try std.testing.expect(!filesystem.isLocal("test://bucket/object")); + + for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors", "file:///tmp/model.safetensors" }) |path| { + try std.testing.expectEqual(ReadHints{ .read_chunk_size = 8 * 1024 * 1024, .high_latency = false }, try filesystem.readHints(path)); + try std.testing.expect(filesystem.isLocal(path)); + } + try std.testing.expectError(error.VFSNotRegistered, filesystem.readHints("missing://bucket/object")); + try std.testing.expect(!filesystem.isLocal("missing://bucket/object")); + try std.testing.expectEqual(@as(?*VFS, &filesystem), VFS.fromIo(filesystem.io())); + try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); +} + +test "Remotes registers every remote scheme with its read hints" { + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + // No credentials and no proxies: nothing is read from the user's home. + var environ: std.process.Environ.Map = .init(std.testing.allocator); + defer environ.deinit(); + var remotes: Remotes = undefined; + try remotes.init(std.testing.allocator, &filesystem, &environ); + defer remotes.deinit(); + + const Case = struct { path: []const u8, local: bool = false, hints: ReadHints }; + const remote: ReadHints = .{ .read_chunk_size = 16 * 1024 * 1024, .high_latency = true }; + const cases = [_]Case{ + .{ .path = "file:///tmp/model", .local = true, .hints = .local }, + .{ .path = "/var/models/model", .local = true, .hints = .local }, + .{ .path = "https://example.com/model", .hints = remote }, + .{ .path = "s3://bucket/model", .hints = remote }, + .{ .path = "gs://bucket/model", .hints = remote }, + .{ .path = "hf://owner/model", .hints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true } }, + }; + for (cases) |case| { + try std.testing.expectEqual(case.hints, try filesystem.readHints(case.path)); + // Only a local file can be read directly; nothing is opened to ask. + try std.testing.expectEqual(case.local, filesystem.isLocal(case.path)); + if (case.local) continue; + // A remote real path carries its scheme once, resolved without a request. + var real_path: [64]u8 = undefined; + try std.testing.expectEqualStrings(case.path, real_path[0..try std.Io.Dir.cwd().realPathFile(filesystem.io(), case.path, &real_path)]); + } +} + +test "VFS serves bare paths and file:// URIs from the inner Io" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const created = try tmp.dir.createFile(testing_io, "bare.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, "bare path bytes", 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + + var uri_buffer: [std.fs.max_path_bytes + 8]u8 = undefined; + const uri = try std.fmt.bufPrint(&uri_buffer, "file://{s}", .{path}); + for ([_][]const u8{ path, uri }) |open_path| { + var real_buffer: [std.fs.max_path_bytes]u8 = undefined; + try std.testing.expectEqualStrings(path, real_buffer[0..try std.Io.Dir.cwd().realPathFile(vfs_io, open_path, &real_buffer)]); + const opened = try std.Io.Dir.openFile(.cwd(), vfs_io, open_path, .{ .mode = .read_only }); + defer opened.close(vfs_io); + const handle, _ = filesystem.getFileHandle(opened); + try std.testing.expectEqual(@as(?usize, null), handle.backend_idx); + // A file this small is never direct. + try std.testing.expect(!filesystem.useDirectIo(opened, .on)); + var contents: [15]u8 = undefined; + try std.testing.expectEqual(contents.len, try opened.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bare path bytes", &contents); + } +} + +test "VFS reads the same bytes under every direct I/O policy" { + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var contents: [3 * alignment + 100]u8 = undefined; + for (&contents, 0..) |*byte, i| byte.* = @truncate(i * 7 + 3); + const created = try tmp.dir.createFile(testing_io, "data.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, &contents, 0); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + const buffer = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(alignment), 2 * alignment); + defer std.testing.allocator.free(buffer); + const plain: []u8 = buffer; + for ([_]DirectIo{ .on, .auto, .off }) |policy| { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + // Direct when the policy and the filesystem allow it, buffered + // otherwise: `off`, or under `auto` a file just written and thus + // cached. The bytes are the same either way. + const direct = filesystem.useDirectIo(file, policy); + if (policy != .on) try std.testing.expect(!direct); + // The decision stands; `off` neither asks nor changes it. + try std.testing.expect(!filesystem.useDirectIo(file, .off)); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // Aligned offset, buffer and length. + @memset(buffer, 0); + try std.testing.expectEqual(buffer.len, try file.readPositionalAll(vfs_io, buffer, alignment)); + try std.testing.expectEqualSlices(u8, contents[alignment..][0..buffer.len], buffer); + + // One aligned call that runs past the end of the file is cut there. + @memset(buffer, 0); + const past_end = try file.readPositional(vfs_io, &.{plain}, 2 * alignment); + try std.testing.expectEqual(contents.len - 2 * alignment, past_end); + try std.testing.expectEqualSlices(u8, contents[2 * alignment ..], buffer[0..past_end]); + try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); + + // A streaming read takes the file off direct reads for good. (A + // misaligned positional read of a direct file does too, with an + // error logged, which the test runner would count; a filesystem + // such as tmpfs may serve it instead, so it is not exercised here.) + var head: [8]u8 = undefined; + const head_slice: []u8 = &head; + try std.testing.expectEqual(head.len, try file.readStreaming(vfs_io, &.{head_slice})); + try std.testing.expectEqualSlices(u8, contents[0..head.len], &head); + try std.testing.expect(!filesystem.useDirectIo(file, policy)); + + // Buffered from now on: unaligned reads and the end of the file. + var tail: [50]u8 = undefined; + try std.testing.expectEqual(tail.len, try file.readPositionalAll(vfs_io, &tail, alignment + 3)); + try std.testing.expectEqualSlices(u8, contents[alignment + 3 ..][0..tail.len], &tail); + try std.testing.expectEqual(@as(usize, 0), try file.readPositional(vfs_io, &.{plain}, contents.len)); + } + + // A descriptor opened for writing is never direct: the flag would + // constrain its writes too. + const writable = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_write }); + defer writable.close(vfs_io); + try std.testing.expect(!filesystem.useDirectIo(writable, .on)); + const writable_handle, _ = filesystem.getFileHandle(writable); + if (comptime direct_io.supported) try std.testing.expectEqual(std.os.linux.E.BADF, direct_io.enable(writable_handle.innerFile().handle)); +} + +test "predictDirectIo answers as useDirectIo would and decides no handle" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + const alignment = 4096; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [3 * alignment + 100]u8 = @splat(0x5a); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "data.bin", .data = &contents }); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "short.bin", .data = contents[0..100] }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(testing_io, &root_buffer)]; + + var filesystem = try VFS.init(std.testing.allocator, testing_io); + defer filesystem.deinit(); + // Another backend's scheme, served here by the plain io: its paths open, + // but they are not local. + try filesystem.registerBackend("remote", .{ .io = testing_io }); + const vfs_io = filesystem.io(); + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/data.bin", .{root}); + + for ([_]DirectIo{ .on, .auto, .off }) |policy| { + // No handle is opened or decided to answer. + const handles = filesystem.handles.len; + const closed_handles = filesystem.closed_handles.items.len; + const predicted = (try filesystem.predictDirectIo(.cwd(), path, policy)).?; + try std.testing.expectEqual(@as(u64, contents.len), predicted.size); + try std.testing.expectEqual(predicted, (try filesystem.predictDirectIo(dir, "data.bin", policy)).?); + try std.testing.expectEqual(handles, filesystem.handles.len); + try std.testing.expectEqual(closed_handles, filesystem.closed_handles.items.len); + + // A fresh handle decides as predicted: direct only under `on`, where + // the filesystem allows it, since the file was just written and so + // is cached. + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + try std.testing.expectEqual(predicted.direct, filesystem.useDirectIo(file, policy)); + if (policy != .on) try std.testing.expect(!predicted.direct); + } + + // A file shorter than a unit is never direct. + const short = (try filesystem.predictDirectIo(dir, "short.bin", .on)).?; + try std.testing.expectEqual(DirectIoPrediction{ .size = 100, .direct = false }, short); + // Nothing to answer for a remote path, though its backend opens the same + // file, or for a missing file. + var remote_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const remote = try std.fmt.bufPrint(&remote_buffer, "remote://{s}/data.bin", .{root}); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(.cwd(), remote, .on)); + try std.testing.expectEqual(null, try filesystem.predictDirectIo(dir, "missing.bin", .on)); +} + +test "auto keeps reading a cold file direct after probing it" { + if (comptime !direct_io.supported) return error.SkipZigTest; + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // One 2 MiB window per sample (`direct_io.residency_samples`): many + // pages each, on 64 KiB-page kernels too. + const window = 2 << 20; + const contents = try std.testing.allocator.alloc(u8, direct_io.residency_samples * window); + defer std.testing.allocator.free(contents); + @memset(contents, 0x5a); + const created = try tmp.dir.createFile(testing_io, "cold.bin", .{ .read = true }); + try created.writePositionalAll(testing_io, contents, 0); + try created.sync(testing_io); + _ = std.os.linux.fadvise(created.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; + created.close(testing_io); + + // The testing io, but the probes draw shifts a quarter window apart, + // 512 KiB from each other's samples. At random shifts a probe may sample + // the pages an earlier one's misses read, and find the file cached + // (`direct_io.predict`): about once in window / page pairs of probes. + const Shifts = struct { + var drawn: u64 = 0; + + fn random(_: ?*anyopaque, buffer: []u8) void { + std.mem.writeInt(u64, buffer[0..8], window / 8 + drawn * window / 4, .native); + drawn += 1; + } + }; + Shifts.drawn = 0; + var vtable = testing_io.vtable.*; + vtable.random = Shifts.random; + const shifting_io: std.Io = .{ .userdata = testing_io.userdata, .vtable = &vtable }; + + var filesystem = try VFS.init(std.testing.allocator, shifting_io); + defer filesystem.deinit(); + const vfs_io = filesystem.io(); + for (0..2) |open| { + // Predicted first, as a reader placing memory before it plans. + const predicted = (try filesystem.predictDirectIo(.cwd(), path, .auto)).?; + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); + defer file.close(vfs_io); + const direct = filesystem.useDirectIo(file, .auto); + // A filesystem that kept the file cached (tmpfs) or refused direct + // reads says nothing here. + if (open == 0 and !direct) return error.SkipZigTest; + try std.testing.expect(predicted.direct); + // The first probes' misses queued reads of the pages they sampled; + // once they landed, sampling the same pages would find the file + // cached. + if (open == 0) try testing_io.sleep(.fromMilliseconds(250), .awake); + try std.testing.expect(direct); + } + // Each probe drew a shift of its own. + try std.testing.expectEqual(@as(u64, 4), Shifts.drawn); +} + +test "standard streams resolve to their own descriptors" { + var vfs: VFS = try .init(std.testing.allocator, std.testing.io); + defer vfs.deinit(); + + const files = [_]std.Io.File{ .stdin(), .stdout(), .stderr() }; + const fds = [_]std.posix.fd_t{ std.posix.STDIN_FILENO, std.posix.STDOUT_FILENO, std.posix.STDERR_FILENO }; + for (files, fds) |file, fd| { + const handle, const backend = vfs.getFileHandle(file); + try std.testing.expectEqual(fd, handle.innerFile().handle); + try std.testing.expect(backend.vtable == vfs.base.inner.vtable); + } +} + test "copy a local file through the VFS" { const allocator = std.testing.allocator; const process_io = std.testing.io; @@ -457,11 +982,9 @@ test "copy a local file through the VFS" { const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); defer allocator.free(destination_uri); - var file_backend: File = .init(allocator, process_io, .{}); - defer file_backend.deinit(); + // `file://` is the VFS's own scheme: no backend to register. var vfs: VFS = try .init(allocator, process_io); defer vfs.deinit(); - try vfs.register("file", file_backend.io()); const vfs_io = vfs.io(); { @@ -481,3 +1004,107 @@ test "copy a local file through the VFS" { defer allocator.free(copied); try std.testing.expectEqualSlices(u8, &source_data, copied); } + +test "creates and deletes on a local path go to the inner Io, from a path or an open directory" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + { + const opened = try std.Io.Dir.cwd().createDirPathOpen(vfs_io, try std.fmt.bufPrint(&uri_buffer, "file://{s}/a/b", .{root}), .{}); + defer opened.close(vfs_io); + // Relative to the descriptor the VFS's handle stands for. + try opened.createDir(vfs_io, "c", .default_dir); + try opened.writeFile(vfs_io, .{ .sub_path = "c/file", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/c/file", .{}); + try opened.deleteFile(vfs_io, "c/file"); + try opened.deleteDir(vfs_io, "c"); + // A colon without "://" is part of a local name. + try opened.writeFile(vfs_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + try tmp.dir.access(testing_io, "a/b/notes:old.txt", .{}); + try opened.deleteFile(vfs_io, "notes:old.txt"); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/notes:old.txt", .{})); + } + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/c", .{})); + try std.Io.Dir.cwd().createDirPath(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try tmp.dir.access(testing_io, "a/d", .{}); + try std.Io.Dir.cwd().deleteDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); + try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/d", .{})); +} + +test "creates and deletes on a remote path fail and write nothing anywhere" { + const allocator = std.testing.allocator; + const testing_io = std.testing.io; + var object: [16]u8 = @splat(0x2a); + var server = try MockServer.init(testing_io, &object, .{}); + defer server.deinit(testing_io); + try server.start(testing_io); + + var client: std.http.Client = .{ .allocator = allocator, .io = testing_io }; + defer client.deinit(); + var endpoint_buffer: [64]u8 = undefined; + const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); + var s3 = try S3.init(allocator, testing_io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); + defer s3.deinit(); + var vfs: VFS = try .init(allocator, testing_io); + defer vfs.deinit(); + try vfs.registerBackend("s3", s3.backend()); + const vfs_io = vfs.io(); + const cwd = std.Io.Dir.cwd(); + + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, "s3://bucket/key", .{ .mode = .read_write })); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, "s3://bucket/key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, "s3://bucket/dir", .default_dir)); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, "s3://bucket/a/b")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, "s3://bucket/a/b", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, "s3://bucket/key")); + try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, "s3://bucket/dir")); + // A directory opened on the remote is no way around it. + const bucket = try cwd.openDir(vfs_io, "s3://bucket", .{}); + defer bucket.close(vfs_io); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.createFile(vfs_io, "key", .{})); + try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "key")); + + // Nothing on local disk, where the base layer would have written, and + // no request to the server. + for ([_][]const u8{ "s3:", "bucket" }) |name| { + try std.testing.expectError(error.FileNotFound, cwd.access(testing_io, name, .{})); + } + try server.stop(testing_io); + try std.testing.expectEqual(0, server.head_requests.load(.acquire) + server.get_requests.load(.acquire)); +} + +test "a name with a colon and no \"://\" is local" { + const testing_io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(testing_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; + + var vfs: VFS = try .init(std.testing.allocator, testing_io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + + const backend_idx, _, _ = try vfs.lookupDir(.cwd(), "notes:old.txt"); + try std.testing.expectEqual(null, backend_idx); + + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + const file = try dir.openFile(vfs_io, "notes:old.txt", .{}); + defer file.close(vfs_io); + var contents: [5]u8 = undefined; + try std.testing.expectEqual(contents.len, try file.readPositionalAll(vfs_io, &contents, 0)); + try std.testing.expectEqualStrings("bytes", &contents); + + var real_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const real_path = real_path_buffer[0..try dir.realPathFile(vfs_io, "notes:old.txt", &real_path_buffer)]; + try std.testing.expect(std.mem.endsWith(u8, real_path, "/notes:old.txt")); +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f52ecdd5..bac383e7 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,7 +78,6 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", - "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -87,13 +86,20 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "io/ShardGrid.zig", + "io/Source.zig", + "io/TensorStore.zig", + "io/backend.zig", + "io/buffered_loader.zig", + "io/direct_loader.zig", + "io/dma_calibration.zig", + "io/host_memory.zig", + "io/loader.zig", + "io/log.zig", + "kernel.zig", "kernels/flash_attention/flash_attention.zig", - "kernels/flash_attention/tests.zig", - "kernels/moe/api.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", - "kernels/moe/weights.zig", - "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", @@ -113,12 +119,11 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", - "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/flashinfer_cutlass", + "//zml/kernels/moe/cutlass_flashinfer", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", @@ -190,6 +195,11 @@ zig_library( "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "//platforms/rocm", + ], + "//conditions:default": [], }), ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 93eafa8d..d1455789 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -23,7 +23,6 @@ const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; -const TensorStore = @import("io.zig").TensorStore; const Compiler = @This(); const log = std.log.scoped(.@"zml/Compiler"); @@ -32,7 +31,6 @@ allocator: std.mem.Allocator, io: std.Io, arena: std.heap.ArenaAllocator, -options: Options, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, @@ -79,7 +77,6 @@ pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, meshes: []const *const Sharding.Mesh = &.{}, - store: ?*const TensorStore = null, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, /// HACK: Extra bytes added to the device memory size reported to XLA. @@ -87,7 +84,8 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate. Remove once XLA properly supports host-pinned inputs. + /// compensate: `zml.io.TensorStore.View.hostMappedBytes` for those the loader delivers + /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options program_name: []const u8 = "zml", @@ -103,7 +101,7 @@ pub const Error = std.mem.Allocator.Error || mlir.Error || upb.SerializeError || pjrtx.Client.CompileError || - error{ MissingDeviceInTile, OpenShardingAtBoundaries }; + error{MissingDeviceInTile}; pub const Donation = union(enum) { implicit: u32, @@ -197,7 +195,6 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, - .options = opts, .partitioner = opts.partitioner orelse .fromTarget(platform.target), .meshes = meshes, .location = unknown_location, @@ -292,7 +289,6 @@ test pushLocation { .mlir_pass_manager = undefined, .mlir_known_types = undefined, .module = undefined, - .options = .{}, .platform = undefined, .partitioner = undefined, .meshes = undefined, @@ -366,13 +362,16 @@ pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mes /// Find a mesh that covers all the given axes pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { if (@TypeOf(logical_axes) != []const Shape.Tag) { - return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); + const comp_tags = comptime Shape.parseTags(logical_axes); + const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); + return compiler.resolveMesh(parsed_tags); } var ok_meshe: ?*const Sharding.Mesh = null; for (compiler.meshes) |shd| { var covers_all: bool = true; - for (logical_axes) |input_axis| { + for (logical_axes) |ax| { + const input_axis = Shape.toTag(ax); var covers_this: bool = false; for (shd.logical.axes.slice()) |existing_axis| { if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { @@ -425,7 +424,10 @@ pub fn compile( ) Error!Exe { return switch (platform.io_impl) { .threaded => compileInternal(allocator, io, platform, func_, args, opts), - .zio => try zio.blockInPlace(struct { + // A compile holds its thread for its whole run, so it takes a pool + // worker of its own instead of queuing behind other jobs; concurrent + // compiles run in parallel. + .zio => try zio.blockInPlaceReserved(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); } @@ -450,7 +452,7 @@ pub fn compileInternal( // in mind we might want to revisit that later. // // This is NOT compatible with std.Io.Evented. - // It is compatible with zio, hence the blockInPlace above. + // It is compatible with zio, hence the blockInPlaceReserved above. const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, @@ -464,7 +466,10 @@ pub fn compileInternal( var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = try compiler.emitMlir(func_, args); + var result = compiler.emitMlir(func_, args) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => unreachable, + }; defer result.output_info.deinit(compiler.allocator); defer result.input_info.deinit(compiler.allocator); @@ -592,21 +597,20 @@ fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTup var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var output_info = blk: { + var result = blk: { compiler.activate(); defer compiler.deactivate(); - - var result = @call(.auto, func_, args); - // Output collection can emit replication constraints, so it is part of tracing. - break :blk try collectOutputInfo(compiler, fn_scope, &result); + break :blk @call(.auto, func_, args); }; + + var output_info = try collectOutputInfo(compiler, fn_scope, &result); errdefer output_info.deinit(compiler.allocator); // Finalize in a separate function that doesn't depend on `func` type. return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); } -fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { const CreateBlockArgumentsCb = struct { compiler: *Compiler, scope: *Scope, @@ -614,23 +618,12 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ O current_argument_id: u32 = 0, infos: std.MultiArrayList(TensorInfo) = .empty, - fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { + fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { // Declare the program argument as a packed u8 const og_shape = tensor._shape; - if (og_shape.containsPartitionSpec(.open)) { - log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); - return error.OpenShardingAtBoundaries; - } const packed_shape = og_shape.packedShape(); - const mlir_ctx = ctx.compiler.mlir_ctx; - const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); - var location: *const mlir.Location = ctx.compiler.unknown_location; - if (ctx.compiler.options.store) |store| { - if (store.id_to_sources.get(tensor.id)) |sources| { - location = location.named(mlir_ctx, sources.tensors[0].name); - } - } - var value = ctx.scope.block.addArgument(mlir_type, location); + const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); + var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); // But immediately create the MLIR value corresponding to the fp4 data. // That way the user code only sees a proper fp4 tensor. @@ -661,7 +654,7 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ O return context.infos; } -fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { const CollectOutputInfoCb = struct { compiler: *Compiler, scope: *Scope, @@ -669,16 +662,9 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutO infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) !void { - // Outputs with no declared sharding default to replication. Constrain the - // returned value so the partitioner produces a replicated output buffer. - const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); - const og_shape = output.shape(); - if (og_shape.containsPartitionSpec(.open)) { - log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); - return error.OpenShardingAtBoundaries; - } - var value = ctx.scope.id_to_argument.get(output.id) orelse - output._value orelse + const og_shape = tensor.shape(); + var value = ctx.scope.id_to_argument.get(tensor.id) orelse + tensor._value orelse @panic("no value found for output tensor"); if (tensor.dtype().bitSizeOf() < 8) { @@ -820,7 +806,6 @@ fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { // even if later it means that some mesh axis is never used. // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas // TODO(#914): decide if we should forbid this altogether - // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); } log.debug("Using {f}", .{m}); @@ -1034,7 +1019,6 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load platform.pjrt_client, platform.pjrt_api, compiler.allocator, - compiler.io, compiler.module, try upb.serialize(options, upb_arena), ); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 040216bc..bc5e6afe 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -21,22 +21,10 @@ mesh: ?*const Mesh, partition: Partitioning, pub const MAX_MESH_RANK = 4; - -/// Full replication across participating devices. -/// For buffers, each device holds a complete copy. Applied as a graph constraint, -/// this closes every dimension to partitioning. `Shape.init` uses this sharding. -/// Unconstrained graph results use `open` instead. pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; -/// Unspecified sharding for graph results. During compilation, the partitioner -/// may infer sharding from surrounding constraints or leave the value replicated. -/// Executable inputs require closed sharding. An executable output with this -/// unspecified sharding defaults to replication. An explicitly declared output -/// sharding containing open dimensions is rejected. -pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; - pub fn get(sharding: Sharding, ax: usize) PartitionSpec { - return sharding.partition.get(ax); + return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); } pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { @@ -64,6 +52,7 @@ pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { } pub fn isFullyReplicated(self: Sharding) bool { + if (self.mesh == null) return true; for (self.partition.toArray()) |spec| { if (spec != .replicated and spec != .out_of_bound) return false; } @@ -109,18 +98,8 @@ test isCompatible { } } -/// Formats the declared sharding, not the partitioner's inferred sharding. -/// Without a mesh, prints `{replicated}` for full replication or `{open}` for -/// unspecified graph sharding. -/// -/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: -/// - `data`: split this dimension over the logical mesh axis named `data`. -/// - `replicated`: keep this dimension whole on each shard. -/// - `?`: leave this dimension to partitioner inference during compilation. -/// - `@dp`: the mesh name. -/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); + const mesh = self.mesh orelse return writer.writeAll("{replicated}"); const specs = self.partition.toArray(); var rank = specs.len; while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; @@ -138,10 +117,8 @@ pub fn format(self: Sharding, writer: *std.Io.Writer) !void { try writer.print("}}@{s}", .{mesh.name}); } -/// Describes how one tensor dimension is distributed across a logical mesh. -/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. -/// In a graph constraint, `.open` lets the partitioner choose either behavior. -/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. +/// Describes how a given Shape axis behaves inside a mesh. +/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? pub const PartitionSpec = enum(u4) { mesh_axis_0 = 0, mesh_axis_1 = 1, @@ -428,110 +405,122 @@ pub const Partitioning = packed struct { try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); } - /// Parse a struct literal into a partitioning object. - /// The struct passed to partitioning must only have fields names present in `tags`. - /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. - /// The default value is important. - /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. - pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { - const logical = if (@TypeOf(partitioning) == Logical) - partitioning - else - // capture all the comptime logic in Logical. - comptime Logical.parse(partitioning); + /// A reusable, mesh-independent partitioning keyed by tensor axis tags. + /// Values name logical mesh axes (eg `.model`), or are `.replicated` / `.open`. + /// It is resolved into a positional `Partitioning` once the tensor tags and the mesh are known, + /// so the same value can be applied to tensors with reordered axes, or on different meshes. + pub const Logical = struct { + pub const Spec = union(enum) { + /// Sharded along the logical mesh axis with this name. + axis: Shape.Tag, + replicated, + open, - return logical.applyUseAll(tags, mesh, default_value); - } - - pub const Logical = union(enum) { - /// Replicates every dimension, including when used as a graph constraint. - replicated, - mapping: struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, - }, - - const replicated_tag: Shape.Tag = Shape.toTag(.replicated); - const open_tag: Shape.Tag = Shape.toTag(.open); - - pub fn parse(v: anytype) Logical { - const V = @TypeOf(v); - - var src_: Shape.TagsArray = .empty; - var tgt_: Shape.TagsArray = .empty; - - if (comptime stdx.meta.isStruct(V)) { - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); - tgt_.appendAssumeCapacity(@tagName(fv)); - - if (!comptime stdx.meta.isTuple(V)) { - src_.appendAssumeCapacity(field_name); - } - } - return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; + pub fn init(value: anytype) Spec { + const V = @TypeOf(value); + if (V == Spec) return value; + if (V == @EnumLiteral()) return switch (value) { + .replicated => .replicated, + .open => .open, + else => .{ .axis = Shape.toTag(value) }, + }; + if (V == Shape.Tag) return .{ .axis = value }; + stdx.debug.compileError("Partitioning.Logical.Spec expects a logical axis tag, .replicated or .open, got {}", .{V}); } - stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); - } - - test "Logical.parse" { - const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); - } - - test "Logical.replicated" { - const mesh: Mesh = .initTest(.logical, .{.model}); - const tags: []const Shape.Tag = &.{ "a", "b" }; - const logical: Logical = .replicated; - for ([_]PartitionSpec{ .open, .replicated }) |default| { - try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); + pub fn resolve(self: Spec, mesh: *const Mesh) PartitionSpec { + return switch (self) { + .replicated => .replicated, + .open => .open, + .axis => |t| .sharded(@intCast(mesh.resolveLogicalAxis(t) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, t }); + })), + }; } - const unspecified = Logical.parse(.{}); - try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); - } + }; - pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { - const mapping = switch (logical.*) { - .replicated => return .{ .replicated(tags.len), null }, - .mapping => |*mapping_| mapping_, + const Entry = struct { tag: Shape.Tag, spec: Spec }; + + entries: stdx.BoundedArray(Entry, MAX_RANK) = .empty, + /// Spec of the axes without an entry. `null` defers to the default of the caller. + default: ?Spec = null, + + pub const replicated: Logical = .{ .default = .replicated }; + + /// Accepts a `Logical`, `.replicated`, or a struct mapping tensor axis tags to specs, + /// eg `.{ .out = .model, .in = .replicated }`. Null optional specs are skipped. + pub fn init(specs: anytype) Logical { + const T = @TypeOf(specs); + if (T == Logical) return specs; + if (T == @EnumLiteral()) return switch (specs) { + .replicated => Logical.replicated, + else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), }; + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Partitioning.Logical expects a struct of partition specs or .replicated, got {}", .{T}); - var partition_: Partitioning = .repeat(default_value, tags.len); - var missing_src_axis: ?u8 = null; - - for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { - const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - missing_src_axis = @intCast(i); - continue; - }; - - const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) - .replicated - else if (mesh_tag == Logical.open_tag) - .open - else spec: { - const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { - stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); - }; - break :spec .sharded(@intCast(mesh_ax)); - }; - partition_ = partition_.set(shape_ax, spec); + var result: Logical = .{}; + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(specs, field_name); + const spec: ?Spec = if (@typeInfo(@TypeOf(value)) == .optional) (if (value) |v| Spec.init(v) else null) else Spec.init(value); + if (spec) |s| result.entries.append(.{ .tag = Shape.toTag(field_name), .spec = s }) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); } - return .{ partition_, missing_src_axis }; + return result; } - pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { - const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); - - if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); + /// Returns the spec of the given tensor axis, or `default` when it has no entry. + pub fn get(self: Logical, tag_: anytype) ?Spec { + const axis_tag = Shape.toTag(tag_); + for (self.entries.constSlice()) |entry| { + if (std.mem.eql(u8, std.mem.span(entry.tag), std.mem.span(axis_tag))) return entry.spec; } - return p; + return self.default; + } + + /// Resolves the partitioning of a tensor with the given axis tags on `mesh`. + /// Every entry must name one of `tags`. + pub fn resolve(self: Logical, tags: []const Shape.Tag, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + var res: Partitioning = .repeat(if (self.default) |d| d.resolve(mesh) else default_value, tags.len); + for (self.entries.constSlice()) |entry| { + const ax = Shape.axisFromTagMaybe(tags, entry.tag) orelse { + stdx.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), entry.tag }); + }; + res = res.set(ax, entry.spec.resolve(mesh)); + } + return res; + } + + test "Logical resolves by tensor axis name on any mesh" { + const tp: Mesh = .initTest(.tp, .{.model}); + const ep: Mesh = .initTest(.ep, .{ .experts, .model }); + const specs: []const Spec = &.{ .init(.model), .replicated, .open }; + for (specs) |spec| { + const logical: Logical = .init(.{ .out = spec, .in = .replicated }); + const out = logical.get(.out).?; + try std.testing.expectEqual(std.meta.activeTag(spec), std.meta.activeTag(out)); + try std.testing.expectEqual(spec.resolve(&ep), out.resolve(&ep)); + try std.testing.expectEqual(null, logical.get(.batch)); + + const tags: []const Shape.Tag = &.{ "in", "batch", "out" }; + try std.testing.expectEqualSlices( + PartitionSpec, + &.{ .replicated, .open, spec.resolve(&tp) }, + logical.resolve(tags, &tp, .open).toArray()[0..3], + ); + try std.testing.expectEqual(spec.resolve(&ep), logical.resolve(tags, &ep, .replicated).get(2)); + } + try std.testing.expectEqual(PartitionSpec.sharded(1), Spec.init(.model).resolve(&ep)); + } + + test "Logical replicated default and optional specs" { + const tp: Mesh = .initTest(.tp, .{.model}); + const logical: Logical = .init(.replicated); + try std.testing.expect(logical.get(.out).? == .replicated); + try std.testing.expectEqual(Partitioning.replicated(2), logical.resolve(&.{ "a", "b" }, &tp, .open)); + + const maybe: ?Spec = null; + const partial: Logical = .init(.{ .a = maybe, .b = @as(?Spec, .init(.model)) }); + try std.testing.expectEqual(1, partial.entries.len); + try std.testing.expectEqual(Partitioning.init(&.{ .open, .mesh_axis_0 }), partial.resolve(&.{ "a", "b" }, &tp, .open)); } }; }; @@ -1478,8 +1467,8 @@ pub const LogicalMesh = struct { /// * dimensions: dimension for each axis, -1 infers the size from underlying topology pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { if (@TypeOf(names) != []const Shape.Tag) { - const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); - return .init(parsed_names, intents, dimensions); + const parsed_names = comptime Shape.parseTags(names); + return .init(@as([]const Shape.Tag, parsed_names.slice()), intents, dimensions); } if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); @@ -1515,10 +1504,6 @@ pub const LogicalMesh = struct { return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; } - pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { - return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); - } - pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { const target = std.mem.span(tag); for (self.axes.slice(), self.intents.slice()) |t, i| { @@ -1645,7 +1630,7 @@ pub const Mesh = struct { return .{ .name = @tagName(name_), .logical = .{ - .axes = .init(axes_tags), + .axes = .init(axes_tags.slice()), .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, .dimensions = @splat(-1), }, @@ -1684,17 +1669,13 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } - pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { - return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); - } - pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (self.logical.axes.slice(), 0..) |ax, i| { + for (self.logical.axes.slice(), 0..) |axis, i| { const size = self.grid.axis_sizes[i]; tile_count *= size; - axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(axis), .size = @intCast(size) })); } const replicas = @divExact(self.grid.device_count, tile_count); if (replicas > 1) { @@ -1705,8 +1686,8 @@ pub const Mesh = struct { for (0..self.grid.device_count) |partition_index| { const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; - for (0..self.logical.axes.len) |ax| { - tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; + for (0..self.logical.axes.len) |axis| { + tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; } // Device IDs here are executable partition indices, not PJRT device IDs. ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); @@ -1719,58 +1700,14 @@ pub const Mesh = struct { // There are at most MAX_MESH_RANK logical names, so one candidate is free. inline for (0..MAX_MESH_RANK + 1) |i| { const name = std.fmt.comptimePrint("__replica_{d}", .{i}); - const taken = for (self.logical.axes.slice()) |ax| { - if (std.mem.eql(u8, std.mem.span(ax), name)) break true; + const taken = for (self.logical.axes.slice()) |axis| { + if (std.mem.eql(u8, std.mem.span(axis), name)) break true; } else false; if (!taken) return name; } unreachable; } - /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute - /// referencing this mesh. Entries in `[...]` follow tensor dimension order. - /// Mesh axis indices in the specs resolve to this mesh's logical axis names. - /// The tensor rank and logical mesh rank are independent. - /// - /// Each dimension spec is encoded as follows: - /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over - /// logical mesh axis `i`. The dimension is closed to further partitioning. - /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is - /// closed, so the partitioner cannot assign a mesh axis to it. - /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. - /// It may acquire available mesh axes or remain replicated. - /// Closed therefore means no additional mesh axes, not necessarily replicated. - /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. - /// - /// Dimension replication (`{}`) applies to one tensor dimension. The optional - /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis - /// must carry identical tensor data across its coordinates and cannot be used - /// to split any dimension, including an open one. - /// - /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, - /// and a shape whose dimensions are ordered `batch`, `hidden`: - /// - /// ``` - /// Partitioning: .{ .batch = .data, .hidden = .open } - /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> - /// ``` - /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. - /// The unused `model` axis stays available because an open dimension exists. - /// - /// ``` - /// Partitioning: .{ .batch = .data, .hidden = .replicated } - /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> - /// ``` - /// Both dimensions are closed. Unused logical axes are explicitly replicated, - /// so devices differing only in their `model` coordinate hold the same shard. - /// - /// ``` - /// Partitioning: .{ .batch = .open, .hidden = .replicated } - /// #sdy.sharding<@dp_mp, [{?}, {}]> - /// ``` - /// Only `batch` may acquire axes, and both `data` and `model` are available. - /// Adding `replicated={"data", "model"}` would prevent either axis from being - /// inferred for `batch`, despite its `{?}` entry. pub fn sdyShardingAttrForShape( data: *const Mesh, parent_allocator: std.mem.Allocator, @@ -1782,27 +1719,24 @@ pub const Mesh = struct { const allocator = arena.allocator(); const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); - const has_open_dimension = shape.containsPartitionSpec(.open); - for (0.., dimensions) |i, *d| { - d.* = switch (shape.partition(i)) { + for (0.., dimensions) |ax, *d| { + d.* = switch (shape.partition(ax)) { .out_of_bound => @panic("undefined"), .replicated => .replicated(ctx), .open => .open(ctx, &.{}), else => d: { - const ax = shape.partition(i).meshAxis().?; - used[ax] = true; - break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); + const axis = shape.partition(ax).meshAxis().?; + used[axis] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(axis)))}); }, }; } var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (data.logical.axes.slice(), 0..) |ax, i| { + for (data.logical.axes.slice(), 0..) |axis, i| { tile_count *= data.grid.axis_sizes[i]; - // Open dimensions can acquire unused logical axes during inference. - // Only reserve those axes for replication when every dimension is closed. - if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); + if (!used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -1818,8 +1752,8 @@ pub const Mesh = struct { var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; var tile_count: usize = 1; - for (0..shape.rank()) |i| { - const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; + for (0..shape.rank()) |ax| { + const size = if (shape.partition(ax).meshAxis()) |axis| self.grid.axis_sizes[axis] else 1; tile_shape.appendAssumeCapacity(size); tile_count *= size; } @@ -2096,7 +2030,7 @@ pub const DeviceGrid = struct { for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { if (used.contains(source)) continue; used.insert(source); - const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; + const info = mesh_.physical.axisInfo(source) orelse continue; try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); } } @@ -2839,7 +2773,7 @@ test "mesh: open and replicated dimension mix" { .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"batch\", \"model\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig deleted file mode 100644 index 539f753f..00000000 --- a/zml/attention/Options.zig +++ /dev/null @@ -1,3 +0,0 @@ -/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every -/// key, as in an encoder or a diffusion transformer, and token_index is ignored. -causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index edd9b4ca..49fe947b 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,29 +93,6 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = @import("Options.zig"); - -pub const Partitioning = struct { - const Logical = zml.Sharding.Partitioning.Logical; - - in_specs: struct { - q: Logical, - k: Logical, - v: Logical, - token_index: Logical = .replicated, - /// Omit to replicate backend metadata tensors. - metadata: ?zml.ops.PartitionSpecs(Metadata) = null, - }, - out_specs: Logical, - - /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. - /// Other dimensions, token indices and backend metadata are replicated. - pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { - const heads: Logical = .parse(.{ .h = mesh_axis }); - return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; - } -}; - /// Causal attention as used in transformers. /// /// **Shapes**: @@ -127,46 +104,19 @@ pub const Partitioning = struct { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); -} +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .stablehlo => b: { + // Generate the attention mask. + const seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); -/// `attention`, causal or not depending on `opts`. -pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ - .in_specs = .{ - .q = partitioning.in_specs.q, - .k = partitioning.in_specs.k, - .v = partitioning.in_specs.v, - .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + break :b attn_output; }, - .out_specs = partitioning.out_specs, - }); + .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention), + }; } - -const Attention = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - token_index: zml.Tensor, - metadata: Metadata, - parameters: Parameters, - opts: Options, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - return switch (args.parameters) { - .stablehlo => b: { - const seq_len = args.k.dim(.k); - const mask: ?zml.Tensor = if (args.opts.causal) - zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) - .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) - else - null; - break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); - }, - .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), - }; - } -}; diff --git a/zml/buffer.zig b/zml/buffer.zig index 8714d80f..64d520a7 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,8 +27,8 @@ test { /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. -/// * loading weights from disk directly to the `device zml.aio.loadBuffers` -/// * can be created by calling `HostBuffer.toDevice(platform)`. +/// * filled from a checkpoint by `zml.io.Loader` (docs/learn/loader.md). +/// * created from host data by `fromSlice` or `fromBytes`. pub const Buffer = struct { _platform: *const Platform, _shape: Shape, @@ -56,6 +56,16 @@ pub const Buffer = struct { pub fn devicePtr(self: *const Shard) *anyopaque { return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } + + /// Copies this shard's bytes to `destination`, which must hold the + /// shard's byte size, and waits for the copy. + pub fn toHost(self: *const Shard, io: std.Io, destination: []u8) !void { + const maybe_event = try self._pjrt_buffer.toHostBuffer(self._platform.pjrt_api, destination); + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + } }; pub const ShardIterator = struct { @@ -99,17 +109,6 @@ pub const Buffer = struct { }.deinit, {}, buffers); } - /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. - /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, - /// This only impacts device memory. - pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { - meta.visit(struct { - fn deinit(_: void, x: *Buffer) void { - x.deinit(); - } - }.deinit, {}, buffers); - } - /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; @@ -119,6 +118,8 @@ pub const Buffer = struct { return @intCast(self._shards.len); } + /// One shard per device of the shape's mesh (the platform's replicated + /// mesh when it names none), in that mesh's canonical device order. pub fn shards(self: *const Buffer) ShardIterator { return .{ ._platform = self._platform, @@ -156,7 +157,6 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const byte_size: usize = if (packed_shape.rank() == 0) packed_shape.dtype().sizeOf() else @@ -164,8 +164,8 @@ pub const Buffer = struct { stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); const base_ptr = data_.ptr; - for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (placement.mesh().?.devicesInCanonicalOrder(), 0..) |device, device_index| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device .data = placement.shardPtr(device_index, base_ptr), @@ -248,10 +248,8 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - - for (platform.physical_mesh.devices_in_canonical_order) |device| { - const memory = platform.devices[device.id].memory(opts.memory).?; + for (metadata.placement.mesh().?.devicesInCanonicalOrder()) |device| { + const memory = deviceMemory(platform, device, opts.memory); const args: pjrt.Client.CreateUninitializedBufferArgs = .{ // Change for each device .dst = .{ .memory = memory.pjrt_memory }, @@ -295,14 +293,14 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); const placement = placementOrPanic(self._platform, self._shape); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(shard_index, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; const size_bytes = placement.localShape(self._shape).byteSize(); const destination = sub_slice.data()[0..size_bytes]; - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); @@ -322,9 +320,9 @@ pub const Buffer = struct { var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); defer shard_slice.free(allocator); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + for (self._shards.constSlice(), 0..) |shard, shard_index| { const sub_slice = placement.shardSlice(shard_index, slice); - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); @@ -421,6 +419,41 @@ test "device round-trip" { } } +test "a buffer on a sub-mesh has one shard per device of that mesh" { + const io = std.testing.io; + const allocator = std.testing.allocator; + // Four devices: the accelerator's when it has them, else a CPU platform's + // (none on a host without the CPU plugin, such as aarch64 Linux). + var devices: testing.Devices = undefined; + devices.init(allocator, io, 4) catch return error.SkipZigTest; + defer devices.deinit(); + const platform = devices.platform; + // Two of the four devices, out of the platform's order. + const nodes = [_]Sharding.PhysicalNode{ .device(platform.devices[3]), .device(platform.devices[1]) }; + var physical: Sharding.PhysicalMesh = try .fromTree(allocator, platform.target, .axis(.bus, .tree, &nodes)); + defer physical.deinit(allocator); + const mesh: Sharding.Mesh = try .init("pair", &physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + + var values: [8]u32 = undefined; + for (&values, 0..) |*value, i| value.* = @intCast(i); + const shape = Shape.init(.{ .b = 4, .d = 2 }, .u32).withPartitioning(&mesh, .{ .b = .model }); + var buffer: Buffer = try .fromBytes(io, platform, shape, std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + try testing.expectShards(allocator, io, &buffer, std.mem.sliceAsBytes(&values), .default); + + const read: Slice = try buffer.toSliceAlloc(allocator, io); + defer read.free(allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); + var into: [8]u32 = undefined; + try buffer.toSlice(io, .init(shape, std.mem.sliceAsBytes(&into))); + try std.testing.expectEqualSlices(u32, &values, &into); + + const pinned: Buffer.HostAccessible = try .init(allocator, io, platform, shape, .undonatable); + defer pinned.deinit(allocator); + pinned.view(u32).copyFrom(&values); + try testing.expectShards(allocator, io, &pinned.buffer, std.mem.sliceAsBytes(&values), .host_pinned); +} + pub const PjrtBufferMetadata = struct { shape: Shape, ty: pjrt.BufferType, @@ -446,10 +479,16 @@ pub const PjrtBufferMetadata = struct { } }; +/// The `kind` memory of `device`, a device of a buffer's mesh, which names it +/// by its index in `platform.devices`. Outside `Buffer` for `Buffer.HostAccessible`. +pub fn deviceMemory(platform: *const Platform, device: Sharding.Device, kind: Memory.Kind) *const Memory { + stdx.debug.assert(device.id < platform.devices.len, "Mesh device {d} is not one of the {d} devices of {t}", .{ device.id, platform.devices.len, platform.target }); + return platform.devices[device.id].memory(kind) orelse stdx.debug.panic("Device {d} doesn't have {t} memory", .{ device.id, kind }); +} + fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { - var sh = shape; - if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; - return Sharding.Placement.init(&sh) catch |err| { + const placed = platform.placedShape(shape); + return Sharding.Placement.init(&placed) catch |err| { @branchHint(.cold); std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index b3e82a9c..20d90657 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -17,6 +17,7 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; +const deviceMemory = @import("../buffer.zig").deviceMemory; const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; const Platform = @import("../platform.zig").Platform; const Shape = @import("../shape.zig").Shape; @@ -249,7 +250,7 @@ pub fn init( ._shards = .empty, }; const metadata: PjrtBufferMetadata = .init(platform, shape_); - const devices = platform.physical_mesh.devices_in_canonical_order; + const devices = metadata.placement.mesh().?.devicesInCanonicalOrder(); const api = platform.pjrt_api; var allocations: HostPinnedAllocations = .empty; @@ -259,7 +260,7 @@ pub fn init( errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); for (devices, 0..) |device, shard_index| { - const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const memory = deviceMemory(platform, device, .host_pinned).pjrt_memory; const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; diff --git a/zml/exe.zig b/zml/exe.zig index 32c0ac48..179df29f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -310,7 +310,7 @@ pub const Exe = struct { /// Non-donated inputs remain owned by the caller. pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(output_refs); } @@ -334,7 +334,9 @@ pub const Exe = struct { /// Executes with optional waiting. The caller owns all input handles, including donated inputs. /// Donated input contents cannot be used after the call; output buffers have their own handles. /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. - pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + /// A failed launch writes no outputs; with `wait`, a failed execution destroys the outputs it + /// wrote, so `results_` holds no buffer after any error. + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) pjrt.ApiError!void { var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, .arg_count = arguments.expected_shapes.len, @@ -342,6 +344,7 @@ pub const Exe = struct { }); defer span.end(); + const api = self.platform.pjrt_api; var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; @@ -350,7 +353,7 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, }; - self.exe.execute(self.platform.pjrt_api, .{ + try self.exe.execute(api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, @@ -358,40 +361,22 @@ pub const Exe = struct { // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, - }) catch |err| { - std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }); + + // Every partition's event is destroyed, awaited first when waiting. + var failure: ?pjrt.ApiError = null; + if (events_slice) |slice| for (slice) |e| { + const ev = e orelse continue; + if (opts.wait) ev.await(api, io) catch |err| { + failure = failure orelse err; + }; + ev.deinit(api); }; - - switch (self.platform.target) { - .neuron => { - for (events_slice.?) |e| { - if (e) |ev| { - if (opts.wait) { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - } - ev.deinit(self.platform.pjrt_api); - } - } - }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - ev.deinit(self.platform.pjrt_api); - } - } - }, + if (failure) |err| { + for (results_.flat_buffers.raw_buffers) |buffer| buffer.deinit(api); + return err; } } - - pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { - const exe = try self.exe.executable(self.platform.pjrt_api); - return try exe.getCompiledMemoryStats(self.platform.pjrt_api); - } }; /// A typed executable whose input and output buffer structures are derived from @@ -475,7 +460,7 @@ pub fn FnExe(comptime function_: anytype) type { /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); - self.exe.call(io, self.args, &self.results, opts); + self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); self.exe.deinitDonatedInputs(self.args); self.results.fill(call.outputs); } @@ -625,7 +610,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); - exe.call(io, args, &results, .{ .wait = wait }); + try exe.call(io, args, &results, .{ .wait = wait }); var output = results.get(Buffer); defer output.deinit(); @@ -633,6 +618,8 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { for (value_buffer._shards.constSlice()) |shard| { try std.testing.expect(shard.isDeleted(platform.pjrt_api)); } + // Executing over the deleted input is an error, not a panic. + try std.testing.expect(std.meta.isError(exe.call(io, args, &results, .{ .wait = true }))); value_buffer.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); diff --git a/zml/io.zig b/zml/io.zig index ab8d8e6e..837b1f1e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,1743 +1,32 @@ +//! Checkpoint loading and virtual filesystem integration. The user guide is +//! docs/learn/loader.md; each file's header holds its internals: +//! +//! - io/loader.zig: options and their checks, the set-up task, starts, +//! program compiles and groups, admission, and the one check of every +//! binding before the backends see it (`LoadSpec`). +//! - io/backend.zig: backend selection, the request size rule and the +//! `LoadSpec` contract. +//! - io/TensorStore.zig, io/Source.zig: bindings, views, and the sources +//! that describe a tensor's checkpoint bytes. +//! - io/direct_loader.zig: sizing, per-file planning, FIFO claims, source +//! workers, per-device pumps and transfer completion. +//! - io/ShardGrid.zig: where a destination's logical bytes land. +//! - io/buffered_loader.zig: whole-tensor staging and positional reads. +//! - io/dma_calibration.zig: DMA block selection. +//! - io/host_memory.zig: pinned host arenas and their blocks. +//! - io/log.zig: log scopes. Loader test support is in `zml.testing`. +//! - vfs: local and remote files behind one `std.Io`, the remote error +//! contract, retries and the rate-limit hold (vfs/request.zig). + const std = @import("std"); -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); pub const VFS = @import("vfs"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; -const CreateOptions = @import("platform.zig").CreateOptions; -const Exe = @import("exe.zig").Exe; -const mem = @import("mem.zig"); -const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); -const safetensors = @import("safetensors.zig"); -const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Placement = Sharding.Placement; -const Slice = @import("slice.zig").Slice; -const Tensor = @import("tensor.zig").Tensor; +pub const Loader = @import("io/loader.zig").Loader; +pub const TensorStore = @import("io/TensorStore.zig"); -const log = std.log.scoped(.@"zml/io"); +pub const dma_calibration = @import("io/dma_calibration.zig"); -pub const TensorStore = struct { - pub const Binding = struct { - tensors: []*safetensors.Tensor, - transformed: bool, - memory: Memory.Kind = .default, - }; - - registry: *safetensors.TensorRegistry, - meshes: []const *const Sharding.Mesh, - id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { - var arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .registry = registry, - .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), - .id_to_sources = .empty, - .allocator = allocator, - .arena = arena, - }; - } - - pub fn deinit(self: *TensorStore) void { - self.id_to_sources.deinit(self.allocator); - self.arena.deinit(); - } - - fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { - const gop = try self.id_to_sources.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Id {} already has associated sources", .{id}); - } - errdefer self.id_to_sources.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = sources; - } - - fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { - const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; - return tensor_desc_ptr; - } - - fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { - const entry = self.getPtrFromKey(key) orelse return null; - - const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); - copy.* = entry.*; - - return copy; - } - - fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { - const sources = self.id_to_sources.get(id) orelse return null; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return sources.tensors[0]; - } - - pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - return self.registry.reader(io, key, buffer); - } - - pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const sources = self.id_to_sources.get(id) orelse return error.NotFound; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - - return sources.tensors[0].reader(io, buffer, .{}); - } - - pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { - return self.id_to_sources.get(id); - } - - pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { - const entry_ptr = self.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - - pub fn view(self: *TensorStore) View { - return .{ .store = self }; - } - - pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (store.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - std.debug.panic( - \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.TensorStore.fromRegistry`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(store.meshes) }); - } - - pub const View = struct { - store: *TensorStore, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - pub fn parent(self: *const View) View { - const slice = self.prefix() orelse unreachable; - const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); - var buffer: [256]u8 = undefined; - @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = index + 1, - }; - } - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - pub fn withLayer(self: *const View, index: usize) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - pub fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn hasKey(self: *const View, subkey: []const u8) bool { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return for (self.store.registry.tensors.keys()) |k| { - if (std.mem.startsWith(u8, k, key)) break true; - } else false; - } - - /// Creates a zml.Tensor from a specific entry in the store. - pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); - if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); - } - - const resolved_mesh = self.store.mesh(mesh_); - if (@TypeOf(partitioning) == Sharding.Partitioning) { - return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); - } - // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| - .parse(p_tags, resolved_mesh, .replicated, partitioning) - else - partitioning; - return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); - if (tensor) |t| { - const storage = self.store.id_to_sources.getPtr(t.id); - storage.?.memory = .host_pinned; - } - return tensor; - } - - pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const source = self.store.dupeSource(key) orelse return null; - - const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); - errdefer self.store.arena.allocator().free(sources); - sources[0] = source; - - var shape = source.shape; - if (tags) |user_tags| { - stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); - @memcpy(shape._tags.slice(), user_tags); - } - shape._sharding = sharding; - if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - /// Accepts "null" as tags - fn parseTags(tags: anytype) ?[]const Shape.Tag { - return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); - } - - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { - const arena = self.store.arena.allocator(); - - var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); - defer tensor_list.deinit(arena); - - var buffer: [256]u8 = undefined; - for (sources) |subkey| { - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const tensor = self.store.dupeSource(key) orelse return null; - tensor_list.appendAssumeCapacity(tensor); - } - - const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; - errdefer arena.free(tensors); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); - - return tensor; - } - - pub fn getShape(self: View, subkey: []const u8) ?Shape { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getShape(key); - } - - pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { - var buffer: [256]u8 = undefined; - const key = if (opts.no_prefix) - subkey - else b: { - break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - }; - return self.store.getShape(key); - } - - pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { - var key_buffer: [256]u8 = undefined; - const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getReader(key, io, buffer); - } - - pub fn count(self: View) usize { - var count_: usize = 0; - const prefix_ = self.prefix() orelse ""; - var it = self.store.registry.tensors.iterator(); - while (it.next()) |item| { - const key = item.key_ptr.*; - if (std.mem.startsWith(u8, key, prefix_)) { - count_ += 1; - } - } - return count_; - } - - fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { - const key = std.fmt.bufPrint(buffer, fmt, args) catch - std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); - return key; - } - }; -}; - -pub const Loader = struct { - allocator: std.mem.Allocator, - platform: *const Platform, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - pinned_buffer_pools: []mem.DynamicBufferPool, - group: stdx.Io.LimitedGroup, - bytes_loaded: std.atomic.Value(usize) = .init(0), - delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, - - pub const Opts = struct { - pub const default: Opts = .{ - .parallelism = 1, - .dma_chunks = 2, - .dma_chunk_size = 4096, - }; - parallelism: usize, - dma_chunks: usize, - dma_chunk_size: usize, - }; - - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { - const pool_count = platform.devices.len; - const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); - errdefer allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(allocator, device); - } - - const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); - errdefer allocator.free(buffer_pools); - for (buffer_pools) |*pool_| { - pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); - } - errdefer for (buffer_pools, 0..) |*pool_, i| { - pool_.deinit(dma_allocators[i].allocator()); - }; - - return .{ - .allocator = allocator, - .platform = platform, - .dma_allocators = dma_allocators, - .dma_chunk_size = opts.dma_chunk_size, - .pinned_buffer_pools = buffer_pools, - .group = .init(opts.parallelism), - }; - } - - pub fn deinit(self: *Loader) void { - self.delivered.deinit(self.allocator); - for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); - self.allocator.free(self.pinned_buffer_pools); - self.allocator.free(self.dma_allocators); - } - - pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { - return self.group.await(io); - } - - pub const LoadOpts = struct { - progress: ?*std.Progress.Node = null, - }; - - pub const LoadError = error{TransformedTensorNotDelivered}; - - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { - const tensor_count = meta.count(Tensor, model); - - var arena: std.heap.ArenaAllocator = .init(self.allocator); - defer arena.deinit(); - - const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(buffers, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { - flattened_buffers_[i] = buffer; - } - }.call, .{flattened_buffers}); - - const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { - flattened_tensors_[i] = tensor; - } - }.call, .{flattened_tensors}); - - // Refuse before scheduling anything, so a failure leaves no task in flight. - var missing: usize = 0; - for (flattened_tensors) |tensor| { - const sources = store.getSourcesById(tensor.id) orelse continue; - if (!sources.transformed or self.delivered.contains(tensor.id)) { - continue; - } - - missing += 1; - logNotDelivered(arena.allocator(), tensor, sources.tensors); - } - - if (missing != 0) { - return error.TransformedTensorNotDelivered; - } - - for (flattened_tensors, flattened_buffers) |tensor, buffer| { - if (store.getSourcesById(tensor.id)) |sources| { - if (sources.transformed) continue; - } - - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); - } - } - - fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { - const max_names = 8; - var names: std.Io.Writer.Allocating = .init(arena); - for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { - names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; - } - - if (sources.len > max_names) { - names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; - } - - log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); - } - - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); - } - - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { - const sources = store.getSourcesById(tensor.id) orelse { - std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); - return; - }; - stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { - log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); - unreachable; - }; - } - - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { - log.err("Failed to load tensor {s}: {}", .{ source.name, e }); - loaded.* = false; - return; - }; - loaded.* = true; - } - - fn loadSingleInner( - self: *Loader, - io: std.Io, - source: *safetensors.Tensor, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - opts: LoadOpts, - ) !void { - var reader = try source.reader(io, &.{}, .{}); - defer reader.deinit(); - - var writer = try MemoryWriter.init( - self.allocator, - io, - self.platform, - self.pinned_buffer_pools, - self.dma_allocators, - self.dma_chunk_size, - shape, - buffer, - memory, - ); - defer writer.deinit(self.allocator); - - const scale = 1024; - - if (opts.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - writer.setProgress(&node); - defer writer.setProgress(null); - var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); - const total = try reader.interface.streamRemaining(&progress_writer.interface); - try progress_writer.interface.flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } else { - const total = try reader.interface.streamRemaining(writer.interface()); - try writer.interface().flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } - } - - pub fn loadExecute( - self: *Loader, - arena: std.mem.Allocator, - io: std.Io, - tensor: Tensor, - buffer: *Buffer, - store: *const TensorStore, - exe: *const Exe, - opts: LoadOpts, - ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { - const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; - const buffers = try arena.alloc(Buffer, sources.len); - const loaded = try arena.alloc(bool, sources.len); - @memset(loaded, false); - defer for (buffers, loaded) |*b, l| if (l) b.deinit(); - - var node = if (opts.progress) |progress| p: { - var writer = std.Io.Writer.Allocating.init(arena); - print: { - writer.writer.writeAll("Running executable on ") catch break :print; - for (sources, 0..) |source, i| { - writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; - } - } - break :p progress.start(writer.written(), 1); - } else null; - defer if (node) |*n| n.end(); - - for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); - } - self.group.await(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - }; - - if (std.mem.findScalar(bool, loaded, false)) |_| { - return error.LoadFailed; - } - - var args = try exe.args(arena); - var results = try exe.results(arena); - - args.set(.{buffers}); - exe.call(io, args, &results, .{ .wait = true }); - - buffer.* = results.get(Buffer); - try self.delivered.put(self.allocator, tensor.id, {}); - } -}; - -test "Loader.loadExecute cleans up donated and non-donated inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = @import("testing.zig").env(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); - const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); - defer allocator.free(path); - - const shape: Shape = .init(.{2}, .i32); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); - try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); - defer store.deinit(); - var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); - defer loader.deinit(); - - inline for (.{ false, true }) |donate| { - const Functions = struct { - fn forward(value: Tensor, weight: Tensor) Tensor { - const sum = value.add(weight); - return if (donate) sum.reuseBuffer(value) else sum; - } - }; - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; - const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); - defer exe.deinit(); - try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); - - var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); - defer output.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - try std.testing.expect(loader.delivered.contains(tensor.id)); - } -} - -pub const ProgressWriter = struct { - inner: *std.Io.Writer, - progress: *std.Progress.Node, - interface: std.Io.Writer, - total: usize = 0, - scale: usize, - - pub const InitOpts = struct { - scale: usize = 1, - }; - - pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { - return .{ - .inner = inner_, - .progress = progress_, - .scale = opts.scale, - .interface = .{ - .buffer = inner_.buffer, - .end = inner_.end, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - .sendFile = sendFile, - }, - }, - }; - } - - pub fn pre(self: *ProgressWriter) usize { - self.inner.buffer = self.interface.buffer; - self.inner.end = self.interface.end; - return self.inner.end; - } - - pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { - self.interface.buffer = self.inner.buffer; - self.interface.end = self.inner.end; - const drained_pre = len_pre -| self.interface.end; - self.total += drained_pre + total; - self.progress.setCompletedItems(self.total / self.scale); - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.drain(self.inner, data, splat); - self.post(len_pre, total); - return total; - } - - pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - errdefer self.post(len_pre, 0); - const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); - self.post(len_pre, total); - return total; - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.flush(self.inner); - } - - pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); - const len_pre = self.pre(); - defer self.post(len_pre, 0); - try self.inner.vtable.rebase(self.inner, preserve, capacity); - } -}; - -pub const MemoryWriter = union(enum) { - direct: DirectMemoryWriter, - buffered: BufferedMemoryWriter, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !MemoryWriter { - return switch (platform.target) { - .cuda, .rocm, .oneapi, .furiosa => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), - }, - .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) - std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) - else - .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, - }; - } - - pub fn interface(self: *MemoryWriter) *std.Io.Writer { - return switch (self.*) { - .direct => &self.direct.interface, - .buffered => &self.buffered.interface, - }; - } - - pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { - switch (self.*) { - .direct => self.direct.deinit(), - .buffered => self.buffered.deinit(allocator), - } - } - - pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { - switch (self.*) { - .direct => self.direct.setProgress(progress), - .buffered => {}, - } - } -}; - -pub const BufferedMemoryWriter = struct { - io: std.Io, - platform: *const Platform, - shape: Shape, - buffer: *Buffer, - interface: std.Io.Writer, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { - return .{ - .io = io, - .platform = platform, - .shape = shape, - .buffer = buffer, - .interface = .{ - .buffer = try allocator.alloc(u8, shape.byteSize()), - .vtable = &.{ - .drain = std.Io.Writer.fixedDrain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, - }; - } - - pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { - if (self.interface.buffer.len > 0) { - allocator.free(self.interface.buffer); - } - } - - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - self.buffer.* = Buffer.from( - self.io, - self.platform, - self.shape, - @ptrCast(self.interface.buffer), - .{ .wait = true }, - ) catch return std.Io.Writer.Error.WriteFailed; - } -}; - -const DirectShardWriter = struct { - const EventContext = struct { - self: *DirectShardWriter, - err: ?*pjrt.Error = null, - pjrt_event: *pjrt.Event, - event: std.Io.Event = .unset, - buffer: []u8, - }; - - allocator: std.mem.Allocator, - io: std.Io, - memory: *const Memory, - pool: *mem.DynamicBufferPool, - total: usize, - pjrt_buffer: *pjrt.Buffer, - - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - offset: usize = 0, - - interface: std.Io.Writer, - flip_flop: u1 = 0, - events_contexts: [2]?EventContext = @splat(null), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { - const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); - const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( - memory.platform.pjrt_api, - .{ - .shape_specs = &.{shape_spec}, - .memory = memory.pjrt_memory, - }, - ); - errdefer transfer_manager.deinit(memory.platform.pjrt_api); - - const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); - errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); - - const buf = try pool.get(allocator, io); - - return .{ - .allocator = allocator, - .io = io, - .memory = memory, - .pool = pool, - .total = shape.byteSize(), - .pjrt_buffer = pjrt_buffer, - .transfer_manager = transfer_manager, - .interface = .{ - .buffer = buf[0..@min(buf.len, shape.byteSize())], - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectShardWriter) void { - self.transfer_manager.deinit(self.memory.platform.pjrt_api); - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; - - const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; - if (needs_fresh_buffer) { - try self.interface.flush(); - } - - @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); - self.interface.end += chunk.len; - - const buffer_full = self.interface.end == self.interface.buffer.len; - if (buffer_full) { - try self.interface.flush(); - } - - return chunk.len; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - try self.flushBuffered(); - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - try self.interface.flush(); - } - - fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { - if (self.offset >= self.total) return; - - const pjrt_api = self.memory.platform.pjrt_api; - - const current_buffer = self.interface.buffer; - const buffered = self.interface.buffered(); - - const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; - if (slice.len == 0) return; - const is_last = (self.offset + slice.len) >= self.total; - - const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { - log.err("error when transferring data to device: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - const ctx = &self.events_contexts[@intCast(self.flip_flop)]; - ctx.* = .{ - .self = self, - .buffer = current_buffer, - .pjrt_event = transfer_event, - }; - - transfer_event.onReady(pjrt_api, EventContext, struct { - fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { - ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); - ctx_.err = err; - ctx_.event.set(ctx_.self.io); - } - }.call, &(ctx.*.?)) catch |err| { - log.err("error when setting up transfer completion callback: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - - if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { - defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; - ctx_previous.event.waitUncancelable(self.io); - defer ctx_previous.pjrt_event.deinit(pjrt_api); - if (ctx_previous.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } - - if (is_last) { - defer ctx.* = null; - defer self.interface = .failing; - const ctx_ = &ctx.*.?; - ctx_.event.waitUncancelable(self.io); - defer ctx_.pjrt_event.deinit(pjrt_api); - if (ctx_.err) |e| { - defer e.deinit(pjrt_api); - log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(pjrt_api)), - e.getMessage(pjrt_api), - }); - return std.Io.Writer.Error.WriteFailed; - } - } else { - self.interface.end = 0; - const buf = self.pool.get(self.allocator, self.io) catch |err| { - log.err("unable to get a new buffer from the pool: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; - } - self.flip_flop ^= 1; - self.offset += slice.len; - } -}; - -const ShardProgress = struct { - const scale: usize = 1024; - - node: std.Progress.Node, - label: [32]u8 = undefined, - completed: usize = 0, - - fn set(self: *ShardProgress, completed: usize) void { - self.completed = completed; - self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); - } -}; - -// Dispatch planning bridges two different orders: -// -// 1. Placement traversal emits byte ranges per shard writer, in device order. -// That order is convenient for asking "what bytes belong to this shard?", -// but it does not match how a tensor file is read. -// -// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter -// must therefore consume dispatch spans in increasing global byte offset. -// -// We first collect placement spans, sort them by global byte range, then fold -// identical ranges into one primary dispatch span plus mirror writers. Identical -// ranges represent replicated shard data: the reader provides those bytes once, -// the primary writer receives them zero-copy, and mirrors receive a copy. -const DispatchSpans = struct { - const DispatchSpan = struct { - start: usize, - end: usize, - primary_writer: usize, - mirror_writer_start: usize, - mirror_writer_len: usize, - }; - - const PlacementSpan = struct { - writer_index: usize, - start: usize, - len: usize, - order: usize, - }; - - spans: []DispatchSpan, - mirror_writers: []usize, - - fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { - const placement = try Sharding.Placement.init(&shape); - const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); - - var placement_span_count: usize = 0; - for (0..ordered_devices.len) |device_index| { - placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); - } - - var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); - defer placement_spans.deinit(allocator); - - const byte_strides = shape.computeByteStrides(); - - for (0..ordered_devices.len) |writer_index| { - appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); - } - - std.debug.assert(placement_spans.items.len == placement_span_count); - - var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); - errdefer spans.deinit(allocator); - - var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); - errdefer mirror_writers.deinit(allocator); - - try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); - - const spans_ = try spans.toOwnedSlice(allocator); - errdefer allocator.free(spans_); - - const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); - errdefer allocator.free(mirror_writers_); - - return .{ - .spans = spans_, - .mirror_writers = mirror_writers_, - }; - } - - fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { - allocator.free(self.spans); - allocator.free(self.mirror_writers); - } - - fn deduplicateByRange( - allocator: std.mem.Allocator, - placement_spans: []PlacementSpan, - total_bytes: usize, - spans: *std.ArrayList(DispatchSpan), - mirror_writers: *std.ArrayList(usize), - ) !void { - const SortContext = struct { - fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { - if (lhs.start != rhs.start) return lhs.start < rhs.start; - if (lhs.len != rhs.len) return lhs.len < rhs.len; - return lhs.order < rhs.order; - } - }; - - std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); - - var i: usize = 0; - var cursor: usize = 0; - while (i < placement_spans.len) { - const span = placement_spans[i]; - if (span.start != cursor) return error.NonContiguousShardPlacement; - - const mirror_writer_start = mirror_writers.items.len; - var j = i + 1; - while (j < placement_spans.len) : (j += 1) { - const mirror = placement_spans[j]; - if (mirror.start != span.start or mirror.len != span.len) break; - try mirror_writers.append(allocator, mirror.writer_index); - } - - try spans.append(allocator, .{ - .start = span.start, - .end = span.start + span.len, - .primary_writer = span.writer_index, - .mirror_writer_start = mirror_writer_start, - .mirror_writer_len = j - i - 1, - }); - cursor += span.len; - i = j; - } - - if (cursor != total_bytes) return error.NonContiguousShardPlacement; - } - - fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { - placement_spans.appendAssumeCapacity(.{ - .writer_index = writer_index, - .start = start, - .len = len, - .order = placement_spans.items.len, - }); - } - - fn appendShardPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - shape: Shape, - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - ) void { - if (shape.rank() == 0) { - appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); - return; - } - - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); - } - - fn appendShardAxisPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - axis: usize, - contiguous_axis: usize, - base_start: i64, - ) void { - const slice = slices[axis]; - if (slice.size == 0) return; - - if (axis == contiguous_axis) { - const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); - const span_len: usize = @intCast(slice.size * byte_strides[axis]); - appendPlacementSpan(placement_spans, writer_index, span_start, span_len); - return; - } - - var i: i64 = 0; - while (i < slice.size) : (i += 1) { - const child_start = base_start + (slice.start + i) * byte_strides[axis]; - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); - } - } - - fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { - if (shape.rank() == 0) return 1; - - const contiguous_axis = contiguousSliceAxis(shape, slices); - var count: usize = 1; - for (slices[0..contiguous_axis]) |slice| { - count *= @intCast(slice.size); - } - return count; - } - - fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { - var axis = shape.rank() - 1; - while (axis > 0) { - const slice = slices[axis]; - if (slice.start != 0 or slice.size != shape.dim(axis)) break; - axis -= 1; - } - return axis; - } -}; - -// Direct load writer state machine. -// -// std.Io.Reader -// | -// v -// DirectMemoryWriter.interface.buffer -// | aliases active primary DirectShardWriter DMA buffer -// v -// commitWindow() -// |-- primary: advance DirectShardWriter.interface.end -// |-- mirrors: copy committed bytes with writeAll() -// |-- boundaries: flush full shard buffers / chunk fence -// v -// DirectShardWriter.flushBuffered() -// | -// v -// PJRT AsyncHostToDeviceTransferManager -// -// The public writer never owns a host staging buffer: it exposes a visible -// prefix of a real shard DMA buffer. The visible prefix may extend beyond the -// current dispatch span to coalesce reader requests, but `commitWindow` must -// scatter those bytes before submitting or rotating the active buffer. The -// first active-primary segment is zero-copy; crossed spans and mirrors treat -// the active buffer as scratch and copy into their own shard writers. -// -// `byte_cursor` is the global tensor position. Shard writer `interface.end` is -// local to the current shard DMA buffer. -pub const DirectMemoryWriter = struct { - allocator: std.mem.Allocator, - shard_writers: []DirectShardWriter, - // Global stream-order spans produced from placement; this is the routing table for committed bytes. - dispatch_spans: DispatchSpans, - // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. - span_index: usize = 0, - // Global tensor byte offset already scattered into shard writers. - byte_cursor: usize = 0, - // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. - active_writer_index: usize, - // Start offset of new reader bytes inside the active shard writer buffer. - window_start: usize, - // Logical maximum public alias window before forcing a cross-shard flush fence. - dma_chunk_size: usize, - shard_progress: ?[]ShardProgress = null, - interface: std.Io.Writer, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !DirectMemoryWriter { - var placed_shape = shape; - if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; - const placement = try Sharding.Placement.init(&placed_shape); - const mesh = placement.mesh().?; - const ordered_devices = mesh.devicesInCanonicalOrder(); - var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); - errdefer allocator.free(shard_writers); - - var initialized: usize = 0; - errdefer for (shard_writers[0..initialized]) |*writer| { - writer.deinit(); - }; - - var pjrt_buffers: Buffer.Shards = .empty; - for (ordered_devices, 0..) |device, i| { - const pool = &pools[device.id]; - const shard_dma_allocator = dma_allocators[device.id].allocator(); - const pjrt_mem = platform.devices[device.id].memory(memory).?; - - shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); - initialized += 1; - - pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); - } - - buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - - const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); - errdefer dispatch_spans.deinit(allocator); - - const first_span = dispatch_spans.spans[0]; - const first_writer = &shard_writers[first_span.primary_writer]; - const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); - - return .{ - .allocator = allocator, - .shard_writers = shard_writers, - .dispatch_spans = dispatch_spans, - .active_writer_index = first_span.primary_writer, - .window_start = first_writer.interface.end, - .dma_chunk_size = dma_chunk_size, - .interface = .{ - .buffer = first_writer.interface.buffer[0..first_window], - .end = 0, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectMemoryWriter) void { - self.setProgress(null); - for (self.shard_writers) |*writer| { - writer.deinit(); - } - self.allocator.free(self.shard_writers); - self.dispatch_spans.deinit(self.allocator); - } - - pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { - const parent = progress orelse { - const states = self.shard_progress orelse return; - for (states) |*s| { - s.node.end(); - } - self.allocator.free(states); - self.shard_progress = null; - return; - }; - - std.debug.assert(self.shard_progress == null); - const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; - for (states, self.shard_writers, 0..) |*state, writer, i| { - state.completed = 0; - const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; - const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; - state.node = parent.start(label, total_items); - state.node.setCompletedItems(0); - } - self.shard_progress = states; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - try self.commitWindow(); - if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - - const consumed = consumed: { - var n: usize = 0; - for (data[0 .. data.len - 1]) |bytes| { - const copied = self.writeWindow(bytes); - n += copied; - - if (copied < bytes.len) break :consumed n; - } - - const pattern = data[data.len - 1]; - switch (pattern.len) { - 0 => {}, - 1 => { - const dest = self.interface.buffer[self.interface.end..]; - const count = @min(dest.len, splat); - @memset(dest[0..count], pattern[0]); - - self.interface.end += count; - n += count; - }, - else => for (0..splat) |_| { - const copied = self.writeWindow(pattern); - n += copied; - - if (copied < pattern.len) break; - }, - } - - break :consumed n; - }; - - try self.commitWindow(); - return consumed; - } - - fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { - const dest = self.interface.buffer[self.interface.end..]; - const n = @min(dest.len, bytes.len); - @memcpy(dest[0..n], bytes[0..n]); - self.interface.end += n; - return n; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - // Commit the active public window if one remains - if (self.span_index < self.dispatch_spans.spans.len) { - try self.commitWindow(); - } - - for (self.shard_writers, 0..) |*shard_writer, i| { - try shard_writer.interface.flush(); - if (self.shard_progress) |states| { - states[i].set(shard_writer.total); - } - } - - self.interface = .failing; - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - - try self.commitWindow(); - if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; - } - - // `interface.buffer` is a clipped alias of the active primary shard DMA - // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; - // this function scatters that visible range through dispatch spans. - // - // The leading segment for `active_writer_index` is already in the right - // DMA buffer and is committed by advancing that shard writer. Later spans - // use the same bytes as scratch and are copied before any full buffers are - // submitted or the public alias is rotated. - fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { - // All dispatch spans are consumed; no shard DMA buffer remains valid to - // expose as the public alias until flush marks the writer as failing. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Only bytes after `window_start` are new reader writes. Bytes before - // it may already be pending in the active shard writer. - const source = self.interface.buffer[self.window_start..self.interface.end]; - if (source.len == 0) return; - - // The visible alias window may cross many dispatch spans. Walk the - // global cursor and route each subrange to its primary and mirrors. - var consumed: usize = 0; - while (consumed < source.len) { - if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; - - const span = self.dispatch_spans.spans[self.span_index]; - if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; - - const n = @min(source.len - consumed, span.end - self.byte_cursor); - const chunk = source[consumed..][0..n]; - - // The first span in the active alias is already in the primary DMA - // buffer. Later ranges use the active buffer as scratch and must be - // copied into their real destination writer. - if (span.primary_writer == self.active_writer_index and consumed == 0) { - self.shard_writers[span.primary_writer].interface.end += n; - } else { - const primary_writer = &self.shard_writers[span.primary_writer]; - if (span.primary_writer == self.active_writer_index) { - @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); - primary_writer.interface.end += n; - } else { - try primary_writer.interface.writeAll(chunk); - } - } - if (self.shard_progress) |states| { - states[span.primary_writer].set(states[span.primary_writer].completed + n); - } - - const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; - for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { - const mirror_writer = &self.shard_writers[mirror_writer_index]; - if (mirror_writer_index == self.active_writer_index) { - @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); - mirror_writer.interface.end += n; - } else { - try mirror_writer.interface.writeAll(chunk); - } - if (self.shard_progress) |states| { - states[mirror_writer_index].set(states[mirror_writer_index].completed + n); - } - } - - self.byte_cursor += n; - consumed += n; - if (self.byte_cursor == span.end) { - self.span_index += 1; - } - } - - // All scratch copies are complete now, so full DMA buffers may be - // submitted without invalidating bytes still needed for scatter. - for (self.shard_writers) |*writer| { - if (writer.interface.end == writer.interface.buffer.len) { - try writer.interface.flush(); - } - } - - if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { - for (self.shard_writers) |*writer| { - try writer.interface.flush(); - } - } - - // The last committed window may finish the tensor exactly. Leave an - // empty public window so any further writes fail through std.Io. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Rotate the public alias to the primary writer for the next stream - // span. `window_start` preserves that writer's already-buffered prefix - // so only newly advanced bytes are scattered on the next commit. - const next_span = self.dispatch_spans.spans[self.span_index]; - const next_writer = &self.shard_writers[next_span.primary_writer]; - self.active_writer_index = next_span.primary_writer; - self.window_start = next_writer.interface.end; - - // Publish the next public alias. Its backing memory is the next primary - // shard buffer, but the visible length is allowed to coalesce across - // upcoming spans until a buffer or tensor boundary. - const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; - const buffer_remaining = next_writer.interface.buffer.len - self.window_start; - const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); - const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; - const tensor_remaining = total - self.byte_cursor; - const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); - if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; - - self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; - self.interface.end = self.window_start; - } -}; - -const DirectMemoryWriterDeviceTest = struct { - const WriteMode = enum { - stream_remaining, - writable_slice_greedy, - }; - - pub const Scenario = struct { - name: []const u8, - create_options: CreateOptions, - shape: Shape, - logical_mesh: Sharding.LogicalMesh, - strategy: Sharding.Strategy, - write_mode: WriteMode = .stream_remaining, - writable_slice_min_len: usize = 128, - pool_chunks: usize = 4, - pool_chunk_size: usize = 1 << 20, - memory: Memory.Kind = .default, - }; - - allocator: std.mem.Allocator, - io: std.Io, - - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { - var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; - defer platform.deinit(self.allocator, self.io); - - const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); - std.log.warn("mesh: {f}", .{mesh}); - try self.runDirectMemoryWriter( - platform, - scenario.shape.withPartitioning(&mesh, partitioning), - scenario.write_mode, - scenario.writable_slice_min_len, - scenario.pool_chunks, - scenario.pool_chunk_size, - scenario.memory, - ); - } - - fn runDirectMemoryWriter( - self: DirectMemoryWriterDeviceTest, - platform: *const Platform, - shape: Shape, - write_mode: WriteMode, - writable_slice_min_len: usize, - pool_chunks: usize, - pool_chunk_size: usize, - memory: Memory.Kind, - ) !void { - const slice = try Slice.alloc(self.allocator, shape); - defer slice.free(self.allocator); - - for (slice.items(f32), 0..) |*e, i| { - e.* = @as(f32, @floatFromInt(i)); - } - - const pool_count = platform.devices.len; - const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); - defer self.allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(self.allocator, device); - } - - const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); - defer self.allocator.free(pools); - for (pools) |*pool| { - pool.* = .init(pool_chunks, pool_chunk_size); - } - defer for (pools, 0..) |*pool, i| { - pool.deinit(dma_allocators[i].allocator()); - }; - - var written_buffer: Buffer = undefined; - var writer: DirectMemoryWriter = try .init( - self.allocator, - self.io, - platform, - pools, - dma_allocators, - pool_chunk_size, - shape, - &written_buffer, - memory, - ); - defer writer.deinit(); - defer written_buffer.deinit(); - - switch (write_mode) { - .stream_remaining => { - var reader: std.Io.Reader = .fixed(slice.constData()); - const streamed = try reader.streamRemaining(&writer.interface); - try std.testing.expectEqual(slice.constData().len, streamed); - }, - .writable_slice_greedy => { - var offset: usize = 0; - while (offset < slice.constData().len) { - const min_len = @max(@as(usize, 1), writable_slice_min_len); - const dest = try writer.interface.writableSliceGreedy(min_len); - const to_write = @min(dest.len, slice.constData().len - offset); - if (to_write == 0) return std.Io.Writer.Error.WriteFailed; - @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); - writer.interface.advance(to_write); - offset += to_write; - } - }, - } - - try writer.interface.flush(); - try written_buffer.await(self.io); - - var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); - defer written_slice.free(self.allocator); - try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); - } -}; - -test "DirectMemoryWriter: replicated tensor" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "replicated_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - }, .{ .rows = .replicated, .cols = .replicated }); -} - -test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "batch_model_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }, .{ .batch = .batch, .model = .model }); -} - -test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_folded_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; - }, - }, .{ .model = .model }); -} - -test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto_writable_slice", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .write_mode = .writable_slice_greedy, - .writable_slice_min_len = 64, - .pool_chunk_size = 1024, - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 3D topology folded model + replicated batch" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "topology_3d_folded_model", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, - .cpu = .{ .device_count = 8 }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; - }, - }, .{ .batch = .replicated, .model = .model }); -} - -test "MemoryWriter can produce a host pinned buffer" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "host_pinned", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .memory = .host_pinned, - }, .{ .batch = .replicated, .model = .model }); +test { + std.testing.refAllDecls(@This()); } diff --git a/zml/io/ShardGrid.zig b/zml/io/ShardGrid.zig new file mode 100644 index 00000000..41969285 --- /dev/null +++ b/zml/io/ShardGrid.zig @@ -0,0 +1,352 @@ +//! Where a destination's logical bytes land. Its devices hold the boxes of an +//! even grid over the tensor (`Sharding.Placement` splits every axis into +//! equal shards), each box packed row-major on its devices, and devices +//! holding the same box are one writer set. Box order and logical order agree +//! on a box's elements, so a contiguous logical range lands as one contiguous +//! range of every box it touches: ranges are placed by arithmetic, never by +//! listing rows. +//! +//! For a [4, 8] tensor whose columns are split in two, each letter one +//! element: +//! +//! Logical bytes Box 0 Box 1 +//! a b c d e f g h a b c d e f g h +//! i j k l m n o p i j k l m n o p +//! q r s t u v w x q r s t u v w x +//! y z A B C D E F y z A B C D E F +//! +//! The logical range from c to n lands as c..l on box 0 and e..n on box 1. +//! Runs are the aligned pieces that stay in one box (`run`: a shard of the +//! innermost split axis, four elements here); a worker walks them to place +//! a range's bytes, the planner only counts them. + +const std = @import("std"); + +const constants = @import("../constants.zig"); +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); + +const ShardGrid = @This(); + +const max_rank = constants.MAX_RANK; +/// Every box has at least one device, and a writer mask holds 64. +pub const max_boxes = 64; + +rank: usize, +/// Elements per axis, of the tensor and of one box. +dims: [max_rank]u64 = @splat(1), +shard: [max_rank]u64 = @splat(1), +/// Byte strides, logical and inside a box. +strides: [max_rank]u64 = @splat(0), +box_strides: [max_rank]u64 = @splat(0), +/// A box's index is row-major over the shard counts of the axes. +index_strides: [max_rank]u64 = @splat(0), +element: u64, +total: u64, +box_bytes: u64, +/// Aligned runs of `run` logical bytes each land in one box, contiguous. +run: u64, +box_count: usize, +/// The canonical device indices holding each box. +writers: [max_boxes]u64 = @splat(0), +/// One box: the shape each device holds. +local: Shape, +/// The devices in canonical order, which the `writers` masks index; +/// borrowed from the placed shape's mesh (the platform's when `shape` names +/// none). +devices: []const Sharding.Device, + +/// The grid of `shape` placed on `platform` (`Platform.placedShape`), its +/// sub-byte storage packed (`Shape.packedShape`): what PJRT allocates. +/// `shape` must be non-empty and place on the platform's devices, as the +/// loader's front end checked. +pub fn init(platform: *const Platform, shape: Shape) ShardGrid { + const placed = platform.placedShape(shape).packedShape(); + std.debug.assert(placed.byteSize() != 0); + const placement = Sharding.Placement.init(&placed) catch |err| + std.debug.panic("ShardGrid: {f} does not place, though checked: {s}", .{ placed, @errorName(err) }); + const local = placement.localShape(placed); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + std.debug.assert(devices.len <= max_boxes); + const element: u64 = placed.dtype().sizeOf(); + var grid: ShardGrid = .{ + .rank = placed.rank(), + .element = element, + .total = placed.byteSize(), + .box_bytes = local.byteSize(), + .run = placed.byteSize(), + .box_count = 1, + .local = local, + .devices = devices, + }; + const strides = placed.computeByteStrides(); + var box_stride = element; + var index_stride: u64 = 1; + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + const dim: u64 = @intCast(placed.dim(axis)); + const shard: u64 = @intCast(local.dim(axis)); + grid.dims[axis] = dim; + grid.shard[axis] = shard; + grid.strides[axis] = @intCast(strides.get(axis)); + grid.box_strides[axis] = box_stride; + grid.index_strides[axis] = index_stride; + box_stride *= shard; + const count = dim / shard; + index_stride *= count; + // The innermost split axis cuts the runs. + if (count > 1 and grid.run == grid.total) grid.run = shard * grid.strides[axis]; + } + // A placement holds every box on at least one of its devices. + std.debug.assert(index_stride <= devices.len); + grid.box_count = @intCast(index_stride); + for (0..devices.len) |writer| { + const slices = placement.slices(writer, placed); + var box: u64 = 0; + for (slices.constSlice(), 0..) |slice, i| box += @as(u64, @intCast(slice.start)) / grid.shard[i] * grid.index_strides[i]; + grid.writers[@intCast(box)] |= @as(u64, 1) << @intCast(writer); + } + for (grid.writers[0..grid.box_count]) |mask| std.debug.assert(mask != 0); + return grid; +} + +fn coordinate(self: *const ShardGrid, offset: u64, axis: usize) u64 { + return offset / self.strides[axis] % self.dims[axis]; +} + +/// The box holding logical byte `offset`, and where the byte lands inside it. +pub fn locate(self: *const ShardGrid, offset: u64) struct { usize, u64 } { + var box: u64 = 0; + var at = offset % self.element; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + box += c / self.shard[i] * self.index_strides[i]; + at += c % self.shard[i] * self.box_strides[i]; + } + return .{ @intCast(box), at }; +} + +/// The bytes of box `box` before logical offset `offset`: a logical range +/// `[start, end)` lands on that box at `[rankIn(start), rankIn(end))`. +/// Counted axis by axis, as in a lexicographic comparison. +pub fn rankIn(self: *const ShardGrid, box: usize, offset: u64) u64 { + if (offset >= self.total) return self.box_bytes; + var before: u64 = 0; + for (0..self.rank) |i| { + const c = self.coordinate(offset, i); + const first = @as(u64, box) / self.index_strides[i] % (self.dims[i] / self.shard[i]) * self.shard[i]; + before += @min(c -| first, self.shard[i]) * self.box_strides[i]; + if (c < first or c >= first + self.shard[i]) return before; + } + return before + offset % self.element; +} + +/// The end of the run holding logical byte `offset`, at most `limit`. +pub fn runEnd(self: *const ShardGrid, offset: u64, limit: u64) u64 { + return @min(limit, (offset / self.run + 1) * self.run); +} + +/// The runs with a byte in `[start, end)`, `start < end`. +pub fn runCount(self: *const ShardGrid, start: u64, end: u64) u64 { + return (end + self.run - 1) / self.run - start / self.run; +} + +/// Devices per box: the grid is even, so every box has as many. +pub fn replicas(self: *const ShardGrid) usize { + return @popCount(self.writers[0]); +} + +const testing = struct { + const CreateOptions = @import("../platform.zig").CreateOptions; + + const Scenario = struct { + name: []const u8, + device_count: u32, + physical_mesh: CreateOptions.PhysicalMesh = .auto, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + /// Logical ranges are cut every `step` bytes, unaligned to rows. + step: u64, + }; + + fn run(scenario: Scenario, partitioning: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = scenario.physical_mesh, + .cpu = .{ .device_count = scenario.device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + try expectPlaces(allocator, platform, scenario.shape.withPartitioning(&mesh, partitioning), scenario.step); + } + + /// Walks the runs of every `step`-byte range into the writers' boxes and + /// compares with each device's slices, element by element; checks that + /// `rankIn` bounds the range's bytes on each box and that `runCount` + /// counts the runs. + fn expectPlaces(allocator: std.mem.Allocator, platform: *const Platform, shape: Shape, step: u64) !void { + const grid: ShardGrid = .init(platform, shape); + const placed = platform.placedShape(shape).packedShape(); + const placement: Sharding.Placement = try .init(&placed); + const box_bytes: usize = @intCast(grid.box_bytes); + const total: usize = @intCast(grid.total); + const source = try allocator.alloc(u8, total); + defer allocator.free(source); + for (source, 0..) |*byte, i| byte.* = @truncate(i *% 131 +% 17); + + const expected = try allocator.alloc(u8, grid.devices.len * box_bytes); + defer allocator.free(expected); + const element: usize = @intCast(grid.element); + for (0..grid.devices.len) |writer| { + const slices = placement.slices(writer, placed); + var index: [max_rank]u64 = @splat(0); + var cursor: usize = 0; + while (cursor < box_bytes) : (cursor += element) { + var logical: u64 = 0; + for (0..grid.rank) |i| logical += (@as(u64, @intCast(slices.get(i).start)) + index[i]) * grid.strides[i]; + @memcpy(expected[writer * box_bytes + cursor ..][0..element], source[@intCast(logical)..][0..element]); + var axis = grid.rank; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < grid.shard[axis]) break; + index[axis] = 0; + } + } + } + + const actual = try allocator.alloc(u8, expected.len); + defer allocator.free(actual); + @memset(actual, 0); + var start: u64 = 0; + while (start < grid.total) : (start += step) { + const end = @min(grid.total, start + step); + var low: [max_boxes]u64 = @splat(std.math.maxInt(u64)); + var high: [max_boxes]u64 = @splat(0); + var runs: u64 = 0; + var at = start; + while (at < end) : (runs += 1) { + const stop = grid.runEnd(at, end); + const box, const landing = grid.locate(at); + low[box] = @min(low[box], landing); + high[box] = @max(high[box], landing + stop - at); + var mask = grid.writers[box]; + while (mask != 0) : (mask &= mask - 1) { + const writer: usize = @ctz(mask); + @memcpy(actual[writer * box_bytes + @as(usize, @intCast(landing)) ..][0..@intCast(stop - at)], source[@intCast(at)..@intCast(stop)]); + } + at = stop; + } + try std.testing.expectEqual(runs, grid.runCount(start, end)); + for (0..grid.box_count) |box| { + const first = grid.rankIn(box, start); + const last = grid.rankIn(box, end); + if (high[box] == 0) { + try std.testing.expectEqual(first, last); + } else { + try std.testing.expectEqual(low[box], first); + try std.testing.expectEqual(high[box], last); + } + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } +}; + +test "a grid places replicated tensors, across range boundaries" { + try testing.run(.{ + .name = "replicated", + .device_count = 4, + .shape = Shape.init(.{ .rows = 9, .cols = 257 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 773, + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "a grid places packed sub-byte storage" { + const logical = Shape.init(.{ .rows = 9, .cols = 256 }, .u2); + try testing.run(.{ + .name = "packed_u2", + .device_count = 4, + .shape = logical, + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 131, + }, .{ .rows = .replicated, .cols = .x }); +} + +test "a grid places 1D mirrored and folded sharding" { + try testing.run(.{ + .name = "mirrored_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .rows = 7, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .step = 2053, + }, .{ .rows = .replicated, .model = .model }); + try testing.run(.{ + .name = "folded_1d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + .step = 3001, + }, .{ .model = .model }); +} + +test "a grid places 2D and 3D sharding" { + try testing.run(.{ + .name = "batch_model_2d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + .step = 4093, + }, .{ .batch = .batch, .model = .model }); + try testing.run(.{ + .name = "folded_model_3d", + .device_count = 8, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + .step = 8191, + }, .{ .batch = .replicated, .model = .model }); + // Three axes, the middle and the last split: runs are a quarter row. + try testing.run(.{ + .name = "middle_and_last_3d", + .device_count = 4, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .shape = Shape.init(.{ .a = 3, .b = 4, .c = 8 }, .u16), + .logical_mesh = .mesh(.{ .x = .low_bandwidth, .y = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x, .y = .link_y }), + .step = 37, + }, .{ .a = .replicated, .b = .x, .c = .y }); +} + +test "a grid places a scalar on every device" { + try testing.run(.{ + .name = "scalar", + .device_count = 4, + .shape = Shape.init(.{}, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + .step = 3, + }, .{}); +} diff --git a/zml/io/Source.zig b/zml/io/Source.zig new file mode 100644 index 00000000..ea90111a --- /dev/null +++ b/zml/io/Source.zig @@ -0,0 +1,1349 @@ +//! A tensor the loader assembles from checkpoint bytes by moving them, never +//! by computing: a checkpoint tensor, or slices, reshapes, transposes, +//! broadcasts, paddings, concatenations and stacks of checkpoint tensors. +//! The ops are `zml.Tensor`'s shape ops, with the same names, tags and +//! argument conventions; slices are static and contiguous, and padding is +//! zeros only. +//! +//! A source comes from its store (`View.source`) and its pieces live in +//! the store's arena, where the ops that rewrite them allocate. Nothing is +//! freed before `TensorStore.deinit`, and the store must not move while +//! sources point to it. The ops touch neither the registry nor the +//! bindings. Sources of different stores do not mix: `concatenate`, +//! `stack` and `bindProgram` panic on them. Placement comes last: +//! `withPartitioning` and `as` return a `Placed`, which has no ops and +//! drops its placement with `unplaced`. A `Placed` is what binds: +//! `Placed.bind` records the model tensor the loader fills from it, +//! `bindProgram` one a program computes from several. Sources are built +//! during model init. +const std = @import("std"); + +const stdx = @import("stdx"); + +const constants = @import("../constants.zig"); +const DataType = @import("../dtype.zig").DataType; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const backend = @import("backend.zig"); +const TensorStore = @import("TensorStore.zig"); + +const Source = @This(); + +store: *TensorStore, +/// Always replicated: placement comes last (`Placed`). +_shape: Shape, +/// Disjoint lattices of elements read from checkpoint tensors; elements no +/// piece covers are zero. zml/io's own: a binding keeps them (`Placed`) +/// and the loader lowers them to bytes (`Placed.lower`). +_pieces: []const Piece, + +/// Checkpoint bits read through an affine lattice: for every `j` with +/// `j[k] < dims[k].count`, the element at row-major index +/// `origin + sum(j[k] * dims[k].step)` of the source's shape is the element +/// at bit `offset + sum(j[k] * dims[k].stride)` of `tensor`. Reshapes keep +/// the row-major order, so they never touch a piece. +const Piece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + /// Outermost first, none of count 1. + dims: []const Dim, +}; + +const Dim = struct { + count: u64, + /// Row-major elements of the source's shape between two steps. + step: u64, + /// Bits of the checkpoint tensor between two steps; zero repeats them. + stride: u64, +}; + +/// `pad`'s per-axis options, as for `Tensor.pad`. +pub const Pad = Tensor.Pad; + +/// The checkpoint tensor `subkey` of `view`, with `tags` (null keeps the +/// checkpoint's); null when the checkpoint has no such tensor. +pub fn init(view: TensorStore.View, subkey: []const u8, tags: ?[]const Shape.Tag) ?Source { + const tensor = view.get(subkey) orelse return null; + const store = view.store; + var shape_ = tensor.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape_.rank(), "checkpoint tensor {s} has shape {f}, but got {d} tags: {f}", .{ tensor.name, shape_, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape_._tags.slice(), user_tags); + } + shape_._sharding = .replicated; + const arena = store.arena.allocator(); + const elements: u64 = shape_.count(); + const pieces: []const Piece = if (elements == 0) &.{} else blk: { + const dims_: []const Dim = if (elements == 1) + &.{} + else + arena.dupe(Dim, &.{.{ .count = elements, .step = 1, .stride = shape_.dtype().bitSizeOf() }}) catch oom(); + break :blk arena.dupe(Piece, &.{.{ .tensor = tensor, .offset = 0, .origin = 0, .dims = dims_ }}) catch oom(); + }; + return .{ .store = store, ._shape = shape_, ._pieces = pieces }; +} + +pub fn shape(self: Source) Shape { + return self._shape; +} + +pub fn dim(self: Source, axis_: anytype) i64 { + return self._shape.dim(axis_); +} + +pub fn dims(self: *const Source) []const i64 { + return self._shape.dims(); +} + +pub fn dtype(self: Source) DataType { + return self._shape.dtype(); +} + +pub fn rank(self: Source) u4 { + return self._shape.rank(); +} + +pub fn axis(self: Source, axis_: anytype) u3 { + return self._shape.axis(axis_); +} + +pub fn count(self: Source) usize { + return self._shape.count(); +} + +pub fn format(self: Source, writer: *std.Io.Writer) !void { + try writer.print("Source({f}, {d} pieces)", .{ self._shape, self._pieces.len }); +} + +// Reshapes: the row-major order is unchanged, so are the pieces. + +pub fn withTags(self: Source, tags: anytype) Source { + return self.reshaped(self._shape.withTags(tags)); +} + +pub fn rename(self: Source, renames: anytype) Source { + return self.reshaped(self._shape.rename(renames)); +} + +pub fn renameTag(self: Source, old: anytype, new: anytype) Source { + return self.reshaped(self._shape.setTag(old, new)); +} + +pub fn splitAxis(self: Source, ax: anytype, split_shape: anytype) Source { + return self.reshaped(self._shape.splitAxis(ax, split_shape)); +} + +pub fn merge(self: Source, merges: anytype) Source { + return self.reshaped(self._shape.mergeAxes(merges)); +} + +pub fn reshape(self: Source, output_shape: anytype) Source { + return self.reshaped(self._shape.reshape(output_shape)); +} + +pub fn flatten(self: Source) Source { + return self.reshape(.{self.count()}); +} + +pub fn insertAxes(self: Source, axis_: anytype, tags: anytype) Source { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); + var res = self._shape; + const ones: [constants.MAX_RANK]i64 = @splat(1); + res._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + return self.reshaped(res); +} + +pub fn appendAxes(self: Source, tags: anytype) Source { + return self.insertAxes(.last, tags); +} + +pub fn squeeze(self: Source, axis_: anytype) Source { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + return self.reshaped(self._shape.remove(a)); +} + +fn reshaped(self: Source, new_shape: Shape) Source { + stdx.debug.assert(new_shape.count() == self.count(), "Source reshape from {f} to {f} changes the element count", .{ self._shape, new_shape }); + var res = self; + res._shape = new_shape.withDtype(self.dtype()); + res._shape._sharding = .replicated; + return res; +} + +/// Reinterprets the bits as `dt`, like `Tensor.bitCast`: to a narrower type, +/// each element becomes the elements of a new last axis `.bitcast`; to a +/// wider one, the last axis must hold one element of `dt`. +pub fn bitCast(self: Source, dt: DataType) Source { + const src_bits: u64 = self.dtype().bitSizeOf(); + const tgt_bits: u64 = dt.bitSizeOf(); + if (src_bits == tgt_bits) return .{ .store = self.store, ._shape = self._shape.withDtype(dt), ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + if (src_bits > tgt_bits) { + const ratio = std.math.divExact(u64, src_bits, tgt_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const res_shape = self._shape.append(.{ .bitcast = @as(i64, @intCast(ratio)) }).withDtype(dt); + for (self._pieces) |piece| { + var flat: Flat = .fromPiece(piece); + flat.origin *= ratio; + for (flat.dims.slice()) |*d| d.step *= ratio; + flat.dims.append(.{ .count = ratio, .step = 1, .stride = tgt_bits }) catch tooManyDims(); + ctx.emitFlat(flat); + } + return ctx.finish(res_shape); + } + const ratio = std.math.divExact(u64, tgt_bits, src_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); + const last: u3 = @intCast(self.rank() - 1); + stdx.debug.assert(self.dim(last) == @as(i64, @intCast(ratio)), "bitCast from {f} to {t} expects a last axis of {d} elements", .{ self._shape, dt, ratio }); + const res_shape = self._shape.remove(last).withDtype(dt); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + // The elements merged into one must be one contiguous run of bits. + const inner: ?usize = for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis == last) break k; + } else null; + const whole = axis_piece.origin[last] == 0 and inner != null and blk: { + const d = axis_piece.dims.get(inner.?); + break :blk d.count == ratio and d.step == 1 and d.stride == src_bits; + }; + if (!whole) std.debug.panic("bitCast of {f} to {t}: an element of {t} would gather bits that are not contiguous in the checkpoint", .{ self._shape, dt, dt }); + _ = axis_piece.dims.orderedRemove(inner.?); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + +// Ops that move elements. + +/// Like `Tensor.slice`, for a static, contiguous slice (`step` 1). +pub fn slice(self: Source, axis_: anytype, s: Tensor.Slice) Source { + stdx.debug.assert(s.dyn_start == null, "Source.slice takes static slices, got {f}", .{s}); + stdx.debug.assert(s.step == 1, "Source.slice takes contiguous slices, got {f}", .{s}); + const a = self.axis(axis_); + const range = s.absolute(self.dim(a)); + const lo: u64 = @intCast(range.start); + const hi: u64 = @intCast(range.end); + const res_shape = self._shape.setDim(a, range.end - range.start); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| ctx.clip(res_shape, axis_piece, a, lo, hi); + } + const sliced = ctx.finish(res_shape); + return if (s.singleton) sliced.reshaped(res_shape.remove(a)) else sliced; +} + +/// Like `Tensor.split`; the result lives in the store's arena. +pub fn split(self: Source, axis_: anytype, split_sizes: []const i64) []Source { + const a = self.axis(axis_); + var total: i64 = 0; + for (split_sizes) |n| total += n; + stdx.debug.assert(total == self.dim(a), "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ total, self.dim(a) }); + const res = self.store.arena.allocator().alloc(Source, split_sizes.len) catch oom(); + var start: i64 = 0; + for (split_sizes, res) |n, *part| { + part.* = self.slice(a, .{ .start = start, .end = start + n }); + start += n; + } + return res; +} + +pub fn chunkExact(self: Source, axis_: anytype, comptime n_chunks: comptime_int) [n_chunks]Source { + const a = self.axis(axis_); + const size = @divExact(self.dim(a), n_chunks); + var chunks: [n_chunks]Source = undefined; + for (&chunks, 0..) |*chunk, i| { + const start = @as(i64, @intCast(i)) * size; + chunk.* = self.slice(a, .{ .start = start, .end = start + size }); + } + return chunks; +} + +/// Like `Tensor.transpose`: no axes reverses them. +pub fn transpose(self: Source, axes_: anytype) Source { + const given = self._shape.axes(axes_); + const r = self.rank(); + var permutation: [constants.MAX_RANK]u3 = undefined; + if (given.len == 0) { + for (0..r) |i| permutation[i] = @intCast(r - 1 - i); + } else { + stdx.debug.assert(given.len == r, "transpose expects {d} axes for {f}, got {d}", .{ r, self._shape, given.len }); + @memcpy(permutation[0..r], given.constSlice()); + } + var as_i64: [constants.MAX_RANK]i64 = undefined; + for (permutation[0..r], as_i64[0..r]) |p, *q| q.* = p; + const res_shape = self._shape.transpose(as_i64[0..r]); + // Axes of one element move nothing: when the others keep their order, + // the row-major order is unchanged, so are the pieces. + var previous: ?u3 = null; + const moves = for (permutation[0..r]) |p| { + if (self.dim(p) == 1) continue; + if (previous) |q| if (p < q) break true; + previous = p; + } else false; + if (!moves) return self.reshaped(res_shape); + var inverse: [constants.MAX_RANK]u3 = undefined; + for (permutation[0..r], 0..) |p, i| inverse[p] = @intCast(i); + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved = axis_piece; + for (0..r) |i| moved.origin[i] = axis_piece.origin[permutation[i]]; + for (moved.dims.slice()) |*d| d.axis = inverse[d.axis]; + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.pad`, with zero as the only padding value. +pub fn pad(self: Source, padding_value: anytype, paddings: anytype) Source { + stdx.debug.assert(isZero(padding_value), "Source.pad pads with zeros, got {any}", .{padding_value}); + const options = self._shape.parseAxesOptions(Tensor.Pad, paddings, .{}); + var res_shape = self._shape; + var moves = false; + for (options.constSlice(), 0..) |p, a| { + stdx.debug.assert(p.low >= 0 and p.high >= 0 and p.interior >= 0, "Source.pad cannot crop, got {any} on axis {d}", .{ p, a }); + const d = self.dim(a); + res_shape._dims.set(a, p.low + d + @max(d - 1, 0) * p.interior + p.high); + if (p.low != 0 or p.interior != 0 or (a != 0 and p.high != 0)) moves = true; + } + // Rows appended after the last one move no element. + if (!moves) return .{ .store = self.store, ._shape = res_shape, ._pieces = self._pieces }; + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |*axis_piece| { + for (options.constSlice(), 0..) |p, a| { + axis_piece.origin[a] = @as(u64, @intCast(p.low)) + axis_piece.origin[a] * @as(u64, @intCast(p.interior + 1)); + } + for (axis_piece.dims.slice()) |*d| d.step *= @intCast(options.get(d.axis).interior + 1); + ctx.emit(res_shape, axis_piece.*); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.broad`: repeats the elements along the axes `other` adds or +/// grows from 1. The loader reads a few large repetitions once each, and +/// copies many small ones from one read of the source per job. +pub fn broad(self: Source, other: Shape) Source { + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self._shape, other }); + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + const r = self.rank(); + var axes_map: [constants.MAX_RANK]u3 = undefined; + if (r == 0 or r == other.rank()) { + for (0..r) |i| axes_map[i] = @intCast(i); + } else { + for (self._shape.tags(), 0..) |t, i| axes_map[i] = other.axis(t); + } + const res_shape = other.withDtype(self.dtype()); + // The axes the elements repeat along: new ones and those grown from 1. + var repeated: [constants.MAX_RANK]bool = @splat(true); + for (0..r) |i| { + if (self.dim(i) != 1) repeated[axes_map[i]] = false; + } + var ctx: Rewrite = .init(self.store); + defer ctx.deinit(); + for (self._pieces) |piece| { + for (ctx.decompose(self._shape, piece)) |axis_piece| { + var moved: AxisPiece = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset }; + for (0..r) |i| moved.origin[axes_map[i]] = axis_piece.origin[i]; + for (axis_piece.dims.constSlice()) |d| { + var mapped = d; + mapped.axis = axes_map[d.axis]; + moved.dims.append(mapped) catch tooManyDims(); + } + for (0..other.rank()) |b| { + if (!repeated[b] or other.dim(b) == 1) continue; + moved.dims.append(.{ .count = @intCast(other.dim(b)), .axis = @intCast(b), .step = 1, .stride = 0 }) catch tooManyDims(); + } + ctx.emit(res_shape, moved); + } + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.concatenate`; the sources must share every dimension but +/// `axis_`, and the dtype. +pub fn concatenate(sources: []const Source, axis_: anytype) Source { + stdx.debug.assert(sources.len > 0, "concatenate expects at least one source", .{}); + if (sources.len == 1) return sources[0]; + const first = sources[0]; + const a = first.axis(axis_); + var total: i64 = 0; + for (sources) |s| { + stdx.debug.assert(s.store == first.store, "concatenate expects sources of one store", .{}); + const same = s.dtype() == first.dtype() and s.rank() == first.rank() and for (0..first.rank()) |i| { + if (i != a and s.dim(i) != first.dim(i)) break false; + } else true; + stdx.debug.assert(same, "concatenate along axis {d} expects matching shapes, got {f} and {f}", .{ a, first._shape, s._shape }); + total += s.dim(a); + } + const res_shape = first._shape.setDim(a, total); + const outer_stride = elementStrides(res_shape)[a]; + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + var offset: u64 = 0; + for (sources) |s| { + for (s._pieces) |piece| { + if (a == 0) { + // Along the outermost axis, a source's elements follow the previous one's. + var flat: Flat = .fromPiece(piece); + flat.origin += offset * outer_stride; + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |*axis_piece| { + axis_piece.origin[a] += offset; + ctx.emit(res_shape, axis_piece.*); + } + } + offset += @intCast(s.dim(a)); + } + return ctx.finish(res_shape); +} + +/// Like `Tensor.stack`; the sources must have the same shape and tags. +pub fn stack(sources: []const Source, axis_: anytype, tag: anytype) Source { + stdx.debug.assert(sources.len > 0, "stack expects at least one source", .{}); + const first = sources[0]; + for (sources[1..]) |s| { + stdx.debug.assert(s.store == first.store, "stack expects sources of one store", .{}); + stdx.debug.assert(first._shape.eqlWithTags(s._shape), "stack expects source shapes to match, got {f} and {f}", .{ first._shape, s._shape }); + } + const a: u3 = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) @intCast(first.rank()) else first.axis(axis_); + const res_shape = first._shape.insertTag(axis_, @intCast(sources.len), tag); + const r = first.rank(); + var ctx: Rewrite = .init(first.store); + defer ctx.deinit(); + for (sources, 0..) |s, i| { + for (s._pieces) |piece| { + if (a == 0) { + var flat: Flat = .fromPiece(piece); + flat.origin += i * first.count(); + ctx.emitFlat(flat); + continue; + } + for (ctx.decompose(s._shape, piece)) |axis_piece| { + var moved = axis_piece; + std.mem.copyBackwards(u64, moved.origin[a + 1 .. r + 1], axis_piece.origin[a..r]); + moved.origin[a] = i; + for (moved.dims.slice()) |*d| { + if (d.axis >= a) d.axis += 1; + } + ctx.emit(res_shape, moved); + } + } + } + return ctx.finish(res_shape); +} + +/// This source placed on `mesh_` by `partitioning`: a struct keyed by the +/// source's tags (axes it does not name are replicated), or a +/// `Sharding.Partitioning`. `.replicated` places it on every device without +/// a mesh, and takes only `.{}`. Placement comes last: a `Placed` only +/// binds (`Placed.bind`, `bindProgram`) or drops its placement +/// (`Placed.unplaced`). +pub fn withPartitioning(self: Source, comptime mesh_: @EnumLiteral(), partitioning: anytype) Placed { + var placed = self._shape; + if (mesh_ == .replicated) { + const T = @TypeOf(partitioning); + if (comptime !(stdx.meta.isStruct(T) and std.meta.fieldNames(T).len == 0)) + @compileError("Source.withPartitioning(.replicated, ...) takes .{}: a replicated source has no partitioning, got " ++ @typeName(T)); + } else { + placed = self._shape.withPartitioning(self.store.mesh(mesh_), partitioning); + } + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// This source in `placed`, a shape built elsewhere: its dimensions, tags +/// and sharding. Like `reshape`, it moves no element: `placed` has the +/// source's element count and dtype. Placement comes last, as for +/// `withPartitioning`. +pub fn as(self: Source, placed: Shape) Placed { + stdx.debug.assert(placed.dtype() == self.dtype() and placed.count() == self.count(), "Source {f} cannot be placed as {f}", .{ self._shape, placed }); + return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; +} + +/// Makes a model tensor of `output` that `program` computes while loading, +/// `program.forward(output, inputs)` over the placed sources `inputs` in this +/// order (`TensorStore.Program.init`). The loader lands each input as it +/// lands a bound source, placed by its shape, in default device memory; +/// compiles the program once for every tensor of an equal program over the +/// same placed shapes; runs it, and frees the inputs. `forward` returns a +/// tensor of `output`, placed alike, in default device memory. +pub fn bindProgram(inputs: []const Placed, output: Shape, program: anytype) Tensor { + stdx.debug.assert(inputs.len != 0, "Source.bindProgram: a program computing {f} needs an input", .{output}); + const store = inputs[0].store; + for (inputs) |input| { + stdx.debug.assert(input.store == store, "Source.bindProgram: the inputs of {f} come from different stores", .{output}); + } + const arena = store.arena.allocator(); + const kept = arena.dupe(Placed, inputs) catch oom(); + const erased = TensorStore.Program.init(arena, program) catch oom(); + return bindNew(store, output, .{ .compute = .{ .program = erased, .inputs = kept } }); +} + +/// A source with its placement (`withPartitioning`, `as`): the only thing +/// that binds, and what a binding keeps (`TensorStore.Binding`). +pub const Placed = struct { + store: *TensorStore, + /// The placed shape, whose row-major elements the pieces index. + shape: Shape, + /// See `Source._pieces`. + _pieces: []const Piece, + + /// Makes the model tensor the loader fills from this source, in default + /// memory unless `View.setMemory` names another. The binding keeps the + /// source as it is; the loader lowers it to bytes (`lower`). + pub fn bind(self: Placed) Tensor { + return bindNew(self.store, self.shape, .{ .load = .{ .source = self } }); + } + + /// This source without its placement, for code that takes a placed + /// source and reworks it by its placement (a layout per device), then + /// places the result again. + pub fn unplaced(self: Placed) Source { + var shape_ = self.shape; + shape_._sharding = .replicated; + return .{ .store = self.store, ._shape = shape_, ._pieces = self._pieces }; + } + + /// The checkpoint tensor this source is, whole and alone, if it is: one + /// piece that reads all of its bits in order. + pub fn whole(self: Placed) ?*const safetensors.Tensor { + if (self._pieces.len != 1) return null; + const piece = self._pieces[0]; + if (piece.offset != 0 or piece.origin != 0 or piece.tensor.byteSize() != self.shape.byteSize()) return null; + const contiguous = switch (piece.dims.len) { + 0 => self.shape.count() == 1, + 1 => piece.dims[0].count == self.shape.count() and piece.dims[0].step == 1 and piece.dims[0].stride == self.shape.dtype().bitSizeOf(), + else => false, + }; + return if (contiguous) piece.tensor else null; + } + + /// The bytes the loader moves, one part per piece in piece order, in + /// `arena`; none for a source that reads no bytes. The one rule from a + /// source's bits to bytes: each part's dimensions outermost first, the + /// innermost run that is contiguous on both sides as its width. + /// `error.SubBytePosition` when a run starts or ends inside a byte. + pub fn lower(self: Placed, arena: std.mem.Allocator) error{ OutOfMemory, SubBytePosition }![]backend.LoadPart { + const parts = try arena.alloc(backend.LoadPart, self._pieces.len); + for (parts, self._pieces) |*part, piece| part.* = try lowerPiece(arena, self.shape, piece); + return parts; + } +}; + +/// A new tensor of `shape_`, bound to `binding`: the one maker of +/// `store`'s bindings (`Placed.bind`, `bindProgram`), whose pieces are +/// disjoint by construction: the loader relies on that and does not check. +fn bindNew(store: *TensorStore, shape_: Shape, binding: TensorStore.Binding) Tensor { + const tensor: Tensor = .fromShape(shape_); + store.id_to_sources.putNoClobber(store.allocator, tensor.id, binding) catch oom(); + return tensor; +} + +/// `piece` of a source placed in `shape_` as one part (`Placed.lower`), its +/// dimensions allocated with `allocator`, none for one run. +fn lowerPiece(allocator: std.mem.Allocator, shape_: Shape, piece: Piece) error{ OutOfMemory, SubBytePosition }!backend.LoadPart { + const ebits: u64 = shape_.dtype().bitSizeOf(); + var lowered: stdx.BoundedArray(backend.Lattice.Dim, backend.max_part_dims) = .empty; + for (piece.dims) |d| lowered.appendAssumeCapacity(.{ .count = d.count, .src_stride = d.stride, .dst_stride = d.step * ebits }); + var width: u64 = ebits; + if (lowered.len != 0) { + const inner = lowered.get(lowered.len - 1); + if (inner.dst_stride == ebits and inner.src_stride == ebits) { + width = inner.count * ebits; + _ = lowered.pop(); + } + } + const dst_bits = piece.origin * ebits; + var aligned = piece.offset % 8 == 0 and dst_bits % 8 == 0 and width % 8 == 0; + for (lowered.slice()) |*d| { + aligned = aligned and d.src_stride % 8 == 0 and d.dst_stride % 8 == 0; + d.src_stride /= 8; + d.dst_stride /= 8; + } + if (!aligned) return error.SubBytePosition; + return .{ .source = piece.tensor, .lattice = .{ + .src_offset = piece.offset / 8, + .dst_offset = dst_bits / 8, + .width = width / 8, + .dims = if (lowered.len == 0) &.{} else try allocator.dupe(backend.Lattice.Dim, lowered.constSlice()), + } }; +} + +// The machinery. A piece is kept flat; an op that addresses axes rewrites it +// in coordinates of the input shape (`AxisPiece`, every dimension moving one +// axis), moves the coordinates, then flattens it in the result's shape. + +/// A piece while an op rewrites it, its dimensions inline. +const Flat = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: u64, + dims: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty, + + fn fromPiece(piece: Piece) Flat { + var flat: Flat = .{ .tensor = piece.tensor, .offset = piece.offset, .origin = piece.origin }; + flat.dims.appendSlice(piece.dims) catch tooManyDims(); + return flat; + } + + /// The same lattice with dimension `k` cut to `n` steps from step `first`. + fn sub(self: Flat, k: usize, first: u64, n: u64) Flat { + var res = self; + const d = self.dims.get(k); + res.origin += first * d.step; + res.offset += first * d.stride; + res.dims.buffer[k].count = n; + return res; + } +}; + +/// A piece in coordinates of a shape: the element at +/// `origin + sum(j[k] * dims[k].step along dims[k].axis)`. +const AxisPiece = struct { + tensor: *const safetensors.Tensor, + offset: u64, + origin: [constants.MAX_RANK]u64 = @splat(0), + dims: stdx.BoundedArray(AxisDim, backend.max_part_dims) = .empty, +}; + +const AxisDim = struct { + count: u64, + axis: u3, + step: u64, + stride: u64, +}; + +/// The scratch of one op and the pieces it makes. +const Rewrite = struct { + store: *TensorStore, + scratch: std.heap.ArenaAllocator, + work: std.ArrayList(Flat) = .empty, + /// The current piece in coordinates of its shape (`decompose`). + axis_pieces: std.ArrayList(AxisPiece) = .empty, + out: std.ArrayList(Piece) = .empty, + + fn init(store: *TensorStore) Rewrite { + return .{ .store = store, .scratch = .init(store.allocator) }; + } + + fn deinit(self: *Rewrite) void { + self.scratch.deinit(); + } + + fn finish(self: *Rewrite, res_shape: Shape) Source { + var s = res_shape; + s._sharding = .replicated; + return .{ + .store = self.store, + ._shape = s, + ._pieces = self.store.arena.allocator().dupe(Piece, self.out.items) catch oom(), + }; + } + + /// `piece` in coordinates of `shape_`, cut where a dimension would carry + /// from one axis into the next; valid until the next call. + fn decompose(self: *Rewrite, shape_: Shape, piece: Piece) []AxisPiece { + const scratch = self.scratch.allocator(); + self.axis_pieces.clearRetainingCapacity(); + self.work.append(scratch, .fromPiece(piece)) catch oom(); + const strides = elementStrides(shape_); + while (self.work.pop()) |flat| { + switch (fitAxes(shape_, strides, flat)) { + .piece => |axis_piece| self.axis_pieces.append(scratch, axis_piece) catch oom(), + .overflow => |overflowing| self.cut(shape_, strides, flat, overflowing), + } + } + return self.axis_pieces.items; + } + + /// Splits `flat`, which carries past the end of axis `a`, into pieces + /// that do not: at the row boundaries of `a` when its longest dimension + /// there steps regularly through the rows, else one piece per step. + fn cut(self: *Rewrite, shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat, a: u3) void { + const scratch = self.scratch.allocator(); + const n: u64 = @intCast(shape_.dim(a)); + const c = (flat.origin / strides[a]) % n; + var longest: ?usize = null; + var longest_span: u64 = 0; + var total: u64 = 0; + for (flat.dims.constSlice(), 0..) |d, k| { + if (d.count <= 1 or axisOfStep(shape_.rank(), strides, d.step) != a) continue; + const span = (d.count - 1) * (d.step / strides[a]); + total += span; + if (longest == null or span > longest_span) { + longest = k; + longest_span = span; + } + } + const k = longest.?; + const d = flat.dims.get(k); + const s = d.step / strides[a]; + const rest = total - longest_span; + if (n % s == 0 and c % s + rest < s) { + const per_row = n / s; + var j: u64 = 0; + if (c >= s) { + // A partial first row. + const head = @min((n - c + s - 1) / s, d.count); + self.work.append(scratch, flat.sub(k, 0, head)) catch oom(); + j = head; + } + const rows = (d.count - j) / per_row; + if (rows > 0) { + var body = flat.sub(k, j, per_row); + body.dims.append(.{ .count = rows, .step = d.step * per_row, .stride = d.stride * per_row }) catch tooManyDims(); + self.work.append(scratch, body) catch oom(); + j += rows * per_row; + } + if (j < d.count) self.work.append(scratch, flat.sub(k, j, d.count - j)) catch oom(); + return; + } + // An irregular lattice: one piece per step of that dimension. + for (0..d.count) |step_index| { + var single = flat.sub(k, step_index, 1); + _ = single.dims.orderedRemove(k); + self.work.append(scratch, single) catch oom(); + } + } + + /// Emits in `res_shape` the elements of `axis_piece` whose coordinate on + /// `a` is in `[lo, hi)`, at that coordinate minus `lo`. + fn clip(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece, a: u3, lo: u64, hi: u64) void { + var span: u64 = 0; + var outer: ?usize = null; + for (axis_piece.dims.constSlice(), 0..) |d, k| { + if (d.axis != a or d.count <= 1) continue; + span += (d.count - 1) * d.step; + if (outer == null or d.step > axis_piece.dims.get(outer.?).step) outer = k; + } + const o = axis_piece.origin[a]; + if (o >= lo and o + span < hi) { + var kept = axis_piece; + kept.origin[a] = o - lo; + self.emit(res_shape, kept); + return; + } + if (o + span < lo or o >= hi) return; + // Cut along the outermost dimension on `a`: the steps whose whole + // sub-lattice is inside stay one piece, the partial ones recurse. + const k = outer.?; + const d = axis_piece.dims.get(k); + const inner = span - (d.count - 1) * d.step; + const full_lo: u64 = if (o >= lo) 0 else std.math.divCeil(u64, lo - o, d.step) catch unreachable; + const full_hi: u64 = if (o + inner >= hi) 0 else @min(d.count, (hi - 1 - inner - o) / d.step + 1); + if (full_lo < full_hi) { + var kept = axis_piece; + kept.dims.buffer[k].count = full_hi - full_lo; + kept.origin[a] = o + full_lo * d.step - lo; + kept.offset += full_lo * d.stride; + self.emit(res_shape, kept); + } + const touch_lo: u64 = if (o + inner >= lo) 0 else std.math.divCeil(u64, lo - o - inner, d.step) catch unreachable; + const touch_hi: u64 = @min(d.count, (hi - 1 - o) / d.step + 1); + const partial = [_][2]u64{ + .{ touch_lo, @min(full_lo, touch_hi) }, + .{ @max(full_hi, touch_lo, full_lo), touch_hi }, + }; + for (partial) |range| { + var j = range[0]; + while (j < range[1]) : (j += 1) { + var part = axis_piece; + part.origin[a] = o + j * d.step; + part.offset += j * d.stride; + _ = part.dims.orderedRemove(k); + self.clip(res_shape, part, a, lo, hi); + } + } + } + + /// Flattens `axis_piece` in `res_shape` into the result. + fn emit(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece) void { + const strides = elementStrides(res_shape); + var flat: Flat = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset, .origin = 0 }; + for (0..res_shape.rank()) |b| flat.origin += axis_piece.origin[b] * strides[b]; + for (axis_piece.dims.constSlice()) |d| { + if (d.count <= 1) continue; + flat.dims.append(.{ .count = d.count, .step = d.step * strides[d.axis], .stride = d.stride }) catch tooManyDims(); + } + self.emitFlat(flat); + } + + /// Adds `flat` to the result, its dimensions outermost first and merged + /// where one continues the other. + fn emitFlat(self: *Rewrite, flat: Flat) void { + var sorted: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (flat.dims.constSlice()) |d| { + if (d.count > 1) sorted.appendAssumeCapacity(d); + } + std.mem.sort(Dim, sorted.slice(), {}, struct { + fn moreOuter(_: void, lhs: Dim, rhs: Dim) bool { + return lhs.step > rhs.step; + } + }.moreOuter); + var merged: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; + for (sorted.constSlice()) |d| { + if (merged.len != 0) { + const last = &merged.slice()[merged.len - 1]; + if (last.step == d.count * d.step and last.stride == d.count * d.stride) { + last.* = .{ .count = last.count * d.count, .step = d.step, .stride = d.stride }; + continue; + } + } + merged.appendAssumeCapacity(d); + } + const arena = self.store.arena.allocator(); + self.out.append(self.scratch.allocator(), .{ + .tensor = flat.tensor, + .offset = flat.offset, + .origin = flat.origin, + .dims = if (merged.len == 0) &.{} else arena.dupe(Dim, merged.constSlice()) catch oom(), + }) catch oom(); + } +}; + +const AxisFit = union(enum) { + piece: AxisPiece, + /// The axis some coordinate would run past. + overflow: u3, +}; + +/// `flat` in coordinates of `shape_`, if no dimension carries into the next +/// axis: each dimension moves the outermost axis its step is a multiple of. +fn fitAxes(shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat) AxisFit { + const r = shape_.rank(); + var axis_piece: AxisPiece = .{ .tensor = flat.tensor, .offset = flat.offset }; + for (0..r) |b| axis_piece.origin[b] = (flat.origin / strides[b]) % @as(u64, @intCast(shape_.dim(b))); + var extent = axis_piece.origin; + for (flat.dims.constSlice()) |d| { + if (d.count <= 1) continue; + const b = axisOfStep(r, strides, d.step); + const step = d.step / strides[b]; + axis_piece.dims.appendAssumeCapacity(.{ .count = d.count, .axis = b, .step = step, .stride = d.stride }); + extent[b] += (d.count - 1) * step; + } + for (0..r) |b| { + if (extent[b] >= @as(u64, @intCast(shape_.dim(b)))) return .{ .overflow = @intCast(b) }; + } + return .{ .piece = axis_piece }; +} + +fn axisOfStep(r: u4, strides: [constants.MAX_RANK]u64, step: u64) u3 { + for (0..r) |b| { + if (step % strides[b] == 0) return @intCast(b); + } + std.debug.panic("a lattice step of {d} elements in a shape of rank {d}", .{ step, r }); +} + +/// Row-major element strides. +fn elementStrides(shape_: Shape) [constants.MAX_RANK]u64 { + var strides: [constants.MAX_RANK]u64 = @splat(0); + var acc: u64 = 1; + var b: usize = shape_.rank(); + while (b > 0) { + b -= 1; + strides[b] = acc; + acc *= @intCast(shape_.dim(b)); + } + return strides; +} + +fn isZero(value: anytype) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int, .comptime_float, .float => value == 0, + else => false, + }; +} + +fn oom() noreturn { + @panic("OOM"); +} + +fn tooManyDims() noreturn { + std.debug.panic("a Source piece needs more than {d} dimensions", .{backend.max_part_dims}); +} + +// Tests: every op is checked against the same function run as a program +// over `zml.Tensor`, the loader's byte placement replayed on the host. + +const testing = @import("../testing.zig"); +const Buffer = @import("../buffer.zig").Buffer; + +/// A checkpoint of in-memory tensors (the registry points nowhere) and the +/// bytes each holds. +const TestCheckpoint = struct { + store: TensorStore, + bytes: std.StringHashMapUnmanaged([]u8) = .empty, + allocator: std.mem.Allocator, + + const Entry = struct { name: []const u8, shape: Shape }; + + /// Registers `entries` end to end, each holding distinct bytes. + fn init(self: *TestCheckpoint, allocator: std.mem.Allocator, entries: []const Entry) !void { + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var offset: u64 = 0; + for (entries) |entry| { + try registry.registerTensor(.{ .file_uri = "memory", .name = entry.name, .shape = entry.shape, .offset = offset }); + offset += entry.shape.byteSize(); + } + self.* = .{ .store = .init(allocator, ®istry, .{}), .allocator = allocator }; + errdefer self.deinit(); + for (entries, 0..) |entry, index| { + const bytes = try allocator.alloc(u8, entry.shape.byteSize()); + errdefer allocator.free(bytes); + for (bytes, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% index *% 61 +% 1); + try self.bytes.put(allocator, self.store.registry.tensors.getKey(entry.name).?, bytes); + } + } + + fn deinit(self: *TestCheckpoint) void { + var it = self.bytes.valueIterator(); + while (it.next()) |bytes| self.allocator.free(bytes.*); + self.bytes.deinit(self.allocator); + self.store.deinit(); + } + + /// The bytes the loader would assemble for `tensor`, placed on the host: + /// its source lowered as the loader lowers it (`Placed.lower`), the + /// bytes no part covers zero. Fails when a part writes a byte another + /// wrote, so that every byte is written exactly once, by a part or the + /// zero fill: the loader relies on disjoint pieces and does not check + /// them. + fn assemble(self: *TestCheckpoint, tensor: Tensor) ![]u8 { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + const parts = try self.store.bindingOf(tensor.id).?.load.source.lower(arena.allocator()); + const out = try self.allocator.alloc(u8, tensor.shape().byteSize()); + errdefer self.allocator.free(out); + @memset(out, 0); + const written = try arena.allocator().alloc(bool, out.len); + @memset(written, false); + for (parts) |part| { + const src = self.bytes.get(part.source.name).?; + const width: usize = @intCast(part.lattice.width); + var runs: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (runs.next()) |run| { + const at: usize = @intCast(run.dst); + for (written[at..][0..width]) |*byte| { + if (byte.*) return error.DoubleWrite; + byte.* = true; + } + @memcpy(out[at..][0..width], src[@intCast(run.src)..][0..width]); + } + } + return out; + } +}; + +fn interleave(comptime T: type, members: []const T, groups: i64) T { + var parts: [4]T = undefined; + for (members, parts[0..members.len]) |member, *part| part.* = member.splitAxis(.row, .{ .group = groups, .r = -1 }); + return T.concatenate(parts[0..members.len], .r).merge(.{ .row = .{ .group, .r } }); +} + +const qkv_inputs = .{ .{ .row = 4, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; +const expert_inputs = .{ .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; + +/// `build(T, x)` builds a tensor from `x` with shape ops only, the same code +/// for `Tensor` and `Source`; `inputs` holds the dims of each `u16` input. +fn expectCase(comptime inputs: anytype, comptime build: anytype) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const n = inputs.len; + var names: [n][]const u8 = undefined; + var name_buffers: [n][16]u8 = undefined; + var shapes: [n]Shape = undefined; + var entries: [n]TestCheckpoint.Entry = undefined; + inline for (inputs, 0..) |dims_, i| { + names[i] = try std.fmt.bufPrint(&name_buffers[i], "input{d}", .{i}); + shapes[i] = Shape.init(dims_, .u16); + entries[i] = .{ .name = names[i], .shape = shapes[i] }; + } + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &entries); + defer checkpoint.deinit(); + + var sources: [n]Source = undefined; + for (&sources, names) |*s, input_name| s.* = checkpoint.store.view().source(input_name, null); + const bound = build(Source, &sources).withPartitioning(.replicated, .{}).bind(); + const assembled = try checkpoint.assemble(bound); + defer allocator.free(assembled); + + const platform = testing.env(); + var tensors: [n]Tensor = undefined; + for (&tensors, shapes) |*t, shape_| t.* = .fromShape(shape_); + const Program = struct { + fn forward(x: []const Tensor) Tensor { + return build(Tensor, x); + } + }; + var exe = try platform.compileFn(allocator, io, Program.forward, .{&tensors}, .{}); + defer exe.deinit(); + var buffers: [n]Buffer = undefined; + var created: usize = 0; + defer for (buffers[0..created]) |*buffer| buffer.deinit(); + for (&buffers, shapes, names) |*buffer, shape_, input_name| { + buffer.* = try .fromBytes(io, platform, shape_, checkpoint.bytes.get(input_name).?); + created += 1; + } + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{buffers[0..]}); + try exe.call(io, args, &results, .{ .wait = true }); + var output = results.get(Buffer); + defer output.deinit(); + const expected = try output.toSliceAlloc(allocator, io); + defer expected.free(allocator); + + std.testing.expect(bound.shape().eqlWithTags(output.shape())) catch |err| { + std.log.err("Source gives {f}, Tensor {f}", .{ bound.shape(), output.shape() }); + return err; + }; + std.testing.expectEqualSlices(u8, expected.constData(), assembled) catch |err| { + std.log.err("the loader's bytes differ from the program's", .{}); + return err; + }; +} + +test "Source concatenates rows as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 2, .col = 4 }, .{ .row = 4, .col = 4 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .row); + } + }.build); +} + +test "Source concatenates columns as Tensor does" { + try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 3, .col = 2 } }, struct { + fn build(comptime T: type, x: []const T) T { + return T.concatenate(x, .col); + } + }.build); +} + +test "Source interleaves q, k and v by heads as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2); + } + }.build); +} + +test "Source interleaves rows as Tensor does" { + try expectCase(.{ .{ .row = 4, .col = 3 }, .{ .row = 4, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 4); + } + }.build); +} + +test "Source stacks experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.concatenate(&members, .row); + } + }.build); +} + +test "Source interleaves stacked experts as Tensor does" { + try expectCase(expert_inputs, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, 2, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + +test "Source interleaves expert vectors as Tensor does" { + // Per-expert vectors (biases): row r of projection p lands at r * 2 + p. + try expectCase(.{ .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; + return T.stack(&members, .last, .proj).merge(.{ .rows = .{ .row, .proj } }); + } + }.build); +} + +test "Source transposes as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].transpose(.{ .col, .row }); + } + }.build); +} + +test "Source interleaves rows of three as Tensor does" { + // lfm2's in_proj: rows [B; C; x] become b0 c0 x0 b1 c1 x1. + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].splitAxis(.row, .{ .m = 3, .c = 2 }).transpose(.{ .c, .m, .col }).merge(.{ .row = .{ .c, .m } }); + } + }.build); +} + +test "Source slices rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 1, .end = 4 }); + } + }.build); +} + +test "Source slices columns as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.col, .{ .start = 1, .end = 3 }); + } + }.build); +} + +test "Source slices by length as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .{ .start = 2, .len = 3 }); + } + }.build); +} + +test "Source slices an interleave as Tensor does" { + try expectCase(qkv_inputs, struct { + fn build(comptime T: type, x: []const T) T { + return interleave(T, x, 2).slice(.row, .{ .start = 3, .end = 7 }); + } + }.build); +} + +test "Source takes a single row as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].slice(.row, .single(2)); + } + }.build); +} + +test "Source pads as Tensor does" { + try expectCase(.{.{ .row = 4, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 4 }, .col = Tensor.Pad{ .low = 1, .interior = 1 } }); + } + }.build); +} + +test "Source pads rows as Tensor does" { + try expectCase(.{.{ .row = 5, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 3 } }); + } + }.build); +} + +test "Source broadcasts a scalar as Tensor does" { + try expectCase(.{.{}}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].broad(Shape.init(.{ .row = 3, .col = 2 }, .u16)); + } + }.build); +} + +test "Source broadcasts a column as Tensor does" { + // A per-tensor scale repeated over each member's rows. + try expectCase(.{ .{ .row = 1, .col = 1 }, .{ .row = 1, .col = 1 } }, struct { + fn build(comptime T: type, x: []const T) T { + const members = [2]T{ x[0].broad(Shape.init(.{ .row = 4, .col = 1 }, .u16)), x[1].broad(Shape.init(.{ .row = 2, .col = 1 }, .u16)) }; + return interleave(T, &members, 2); + } + }.build); +} + +test "Source bitcasts to a narrower type as Tensor does" { + try expectCase(.{.{ .row = 3, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).merge(.{ .col = .{ .col, .bitcast } }); + } + }.build); +} + +test "Source bitcasts to a wider type as Tensor does" { + try expectCase(.{.{ .row = 2, .col = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].bitCast(.u8).splitAxis(.col, .{ .col = 3, .pair = 2 }).merge(.{ .pair = .{ .pair, .bitcast } }).bitCast(.u32); + } + }.build); +} + +test "Source swizzles block scales as Tensor does" { + // The 128x4 block-scale layout, at 8x4: rows padded to 8 and tiled + // by 2x4, columns padded to 4. + try expectCase(.{.{ .e = 2, .m = 7, .k = 6 }}, struct { + fn build(comptime T: type, x: []const T) T { + return x[0].pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + } + }.build); +} + +test "Source splits then interleaves as Tensor does" { + // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. + try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { + fn build(comptime T: type, x: []const T) T { + const qkv = x[0]; + const members = [4]T{ + qkv.slice(.row, .{ .start = 0, .end = 2 }), + qkv.slice(.row, .{ .start = 2, .end = 4 }), + qkv.slice(.row, .{ .start = 4, .end = 6 }), + x[1], + }; + return interleave(T, &members, 2); + } + }.build); +} + +test "Source chunks then stacks as Tensor does" { + try expectCase(.{.{ .row = 6, .col = 4 }}, struct { + fn build(comptime T: type, x: []const T) T { + const chunks = x[0].chunkExact(.row, 3); + return T.stack(&.{ chunks[2], chunks[0] }, .last, .pair).insertAxes(0, .{.one}).squeeze(0); + } + }.build); +} + +test "a source binds as its pieces, which the loader lowers to byte parts" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{ + .{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }, + .{ .name = "k", .shape = .init(.{ 2, 3 }, .u16) }, + .{ .name = "v", .shape = .init(.{ 2, 3 }, .u16) }, + }); + defer checkpoint.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = checkpoint.store.view(); + const q = view.source("q", .{ .row, .col }); + const entries = [_]*const safetensors.Tensor{ view.get("q").?, view.get("k").?, view.get("v").? }; + + // Relabels keep the whole tensor, one run of it; only `bind` records + // anything, and the binding keeps the placed source as it is. + const relabeled = q.splitAxis(.row, .{ .h = 2, .hd = 2 }).merge(.{ .row = .{ .h, .hd } }).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(@as(usize, 0), checkpoint.store.id_to_sources.count()); + const bound = checkpoint.store.bindingOf(relabeled.bind().id).?.load.source; + try std.testing.expectEqual(relabeled._pieces, bound._pieces); + try std.testing.expectEqual(entries[0], bound.whole().?); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{.whole(entries[0])}, try bound.lower(arena.allocator())); + // One checkpoint tensor read twice is two pieces of it. + const twice = Source.concatenate(&.{ q, view.source("q", .{ .row, .col }) }, .row).withPartitioning(.replicated, .{}); + try std.testing.expectEqual(null, twice.whole()); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .dst_offset = 0, .width = 24 } }, + .{ .source = entries[0], .lattice = .{ .dst_offset = 24, .width = 24 } }, + }, try twice.lower(arena.allocator())); + + // q, k and v in two groups: each member is one part of two runs, its + // rows of a group side by side with the others'. + const members = [_]Source{ q, view.source("k", .{ .row, .col }), view.source("v", .{ .row, .col }) }; + const fused = interleave(Source, &members, 2).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 12, .dims = &.{.{ .count = 2, .src_stride = 12, .dst_stride = 24 }} } }, + .{ .source = entries[1], .lattice = .{ .src_offset = 0, .dst_offset = 12, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + .{ .source = entries[2], .lattice = .{ .src_offset = 0, .dst_offset = 18, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, + }, try fused.lower(arena.allocator())); + + // A transpose is one part of single elements. + const transposed = q.transpose(.{}).withPartitioning(.replicated, .{}); + try std.testing.expectEqualDeep(&[_]backend.LoadPart{ + .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 2, .dims = &.{ + .{ .count = 3, .src_stride = 2, .dst_stride = 8 }, + .{ .count = 4, .src_stride = 6, .dst_stride = 2 }, + } } }, + }, try transposed.lower(arena.allocator())); +} + +test "assembling fails on a byte two pieces write" { + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(std.testing.allocator, &.{.{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }}); + defer checkpoint.deinit(); + // Only a hand-built binding can repeat a piece. + const q = checkpoint.store.view().source("q", .{ .row, .col }); + const pieces = try checkpoint.store.arena.allocator().dupe(Piece, &.{ q._pieces[0], q._pieces[0] }); + const tensor = bindNew(&checkpoint.store, q.shape(), .{ .load = .{ .source = .{ .store = &checkpoint.store, .shape = q.shape(), ._pieces = pieces } } }); + try std.testing.expectError(error.DoubleWrite, checkpoint.assemble(tensor)); +} + +test "a source is placed last, by as or withPartitioning, and binds in its placed shape" { + const allocator = std.testing.allocator; + var checkpoint: TestCheckpoint = undefined; + try checkpoint.init(allocator, &.{.{ .name = "w", .shape = .init(.{ 4, 6 }, .u16) }}); + defer checkpoint.deinit(); + const mesh: Sharding.Mesh = .initTest(.tp, .{.model}); + const transposed = checkpoint.store.view().source("w", .{ .row, .col }).transpose(.{}); + + // `as` takes a shape built elsewhere, here of another rank: its + // dimensions, tags and sharding, for the same elements. + const placed = Shape.init(.{ .flat = 24 }, .u16).withPartitioning(&mesh, .{ .flat = .model }); + const placed_source = transposed.as(placed); + const flat = placed_source.bind(); + try std.testing.expect(flat.shape().eqlWithTagsAndPartitioning(placed)); + try std.testing.expect(checkpoint.store.bindingOf(flat.id).?.load.source.shape.eqlWithTagsAndPartitioning(placed)); + const replicated = transposed.withPartitioning(.replicated, .{}).bind(); + try std.testing.expect(replicated.shape().eqlWithTags(transposed.shape())); + try std.testing.expectEqual(null, replicated.shape()._sharding.mesh); + // `unplaced` keeps the placed dimensions and tags, without the sharding. + const dropped = placed_source.unplaced(); + try std.testing.expect(dropped.shape().eqlWithTags(placed)); + try std.testing.expectEqual(null, dropped.shape()._sharding.mesh); + + const flat_bytes = try checkpoint.assemble(flat); + defer allocator.free(flat_bytes); + const replicated_bytes = try checkpoint.assemble(replicated); + defer allocator.free(replicated_bytes); + try std.testing.expectEqualSlices(u8, replicated_bytes, flat_bytes); +} + +test "lattices stay one piece per source at checkpoint sizes" { + // Registered without bytes: nothing is evaluated. + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "memory", .name = "gate", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "memory", .name = "up", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const view = store.view(); + const members = [_]Source{ view.source("gate", .{ .row, .col }), view.source("up", .{ .row, .col }) }; + + // One row per group: two pieces, each 14336 runs of one row. + const finest = interleave(Source, &members, 14336); + try std.testing.expectEqual(@as(usize, 2), finest._pieces.len); + for (try finest.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 14336), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 8192), part.lattice.width); + } + // One group per device of four: four runs of 28 MiB each. + const per_shard = interleave(Source, &members, 4); + for (try per_shard.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { + try std.testing.expectEqual(@as(u64, 4), part.lattice.runs()); + try std.testing.expectEqual(@as(u64, 3584 * 8192), part.lattice.width); + } + // Slicing through the interleave cuts at most a few pieces. + const sliced = finest.slice(.row, .{ .start = 1001, .end = 20001 }); + try std.testing.expect(sliced._pieces.len <= 6); +} diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig new file mode 100644 index 00000000..3b159bbc --- /dev/null +++ b/zml/io/TensorStore.zig @@ -0,0 +1,558 @@ +//! Associates model tensors with checkpoint sources and prefixed views, +//! over the entries it takes from a checkpoint registry. +//! +//! The store owns those entries, the bindings, and every source and program +//! value (its arena). Views and sources point to it: once `view` was +//! called it must not move, and it must outlive them and every load +//! started from it (`Loader.start` borrows it until `wait` or `cancel`). +//! It borrows the meshes of `Options.meshes`, which belong to the platform. +//! Binding cannot fail: a missing key panics with the key, and what the +//! loader cannot fill is its error (`Loader.start`'s `InvalidBinding`). +const std = @import("std"); + +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; +const MemorySpace = @import("../platform.zig").Memory.Kind; +const Platform = @import("../platform.zig").Platform; +const Quantization = @import("../quantization.zig").Quantization; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); + +const TensorStore = @This(); + +/// Where the loader delivers a loaded tensor, on every device of its mesh +/// (`View.setMemory`). The destination is allocated once per device +/// whatever its pieces; the host kinds take no device memory. +pub const Memory = enum { + /// Device memory. + default, + /// Buffers in each device's pinned host memory space. An executable + /// that takes them as device inputs copies them to the device first. + host_pinned, + /// Pinned host memory that executables read in place: each device's + /// buffer is a device-kind view of it (`createViewOfDeviceBuffer`), + /// and the memory goes with the last view. For a table too large for + /// device memory, of which a step reads a few rows. XLA counts the + /// views against device memory (`View.hostMappedBytes`). + host_mapped, + + /// Whether the bytes live in host memory, written by the CPU. + pub fn isHost(self: Memory) bool { + return switch (self) { + .default => false, + .host_pinned, .host_mapped => true, + }; + } + + /// The memory space of each device's delivered buffer. + pub fn space(self: Memory) MemorySpace { + return switch (self) { + .default, .host_mapped => .default, + .host_pinned => .host_pinned, + }; + } +}; + +/// What the loader reads of a bound tensor (`bindingOf`): the source +/// it fills the tensor from, or the program that computes it. +pub const Binding = union(enum) { + /// The loader fills the tensor from `source` (`Source.Placed.bind`), + /// every byte no piece covers zero. + load: Load, + /// A program computes the tensor while loading (`Source.bindProgram`). + /// Its output lands in default device memory. + compute: Compute, + + pub const Load = struct { + source: Source.Placed, + /// Where the loader delivers this tensor (`View.setMemory`). + memory: Memory = .default, + }; + + pub const Compute = struct { + program: Program, + /// The program's inputs in order, each landed in default device + /// memory, placed by its shape. + inputs: []const Source.Placed, + + /// Whether this computing `output` and `other` computing + /// `other_output` compile to the same executable: equal programs + /// over the same shapes placed alike. + pub fn sameExecutable(self: Compute, output: Shape, other: Compute, other_output: Shape) bool { + if (!self.program.eql(other.program)) return false; + if (!output.eqlWithTagsAndPartitioning(other_output) or self.inputs.len != other.inputs.len) return false; + for (self.inputs, other.inputs) |a, b| { + if (!a.shape.eqlWithTagsAndPartitioning(b.shape)) return false; + } + return true; + } + + /// Compiles the program computing a tensor of `output` from inputs + /// of the shapes it binds, its module named after the program. + pub fn compile( + self: Compute, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + output: Shape, + ) Compiler.Error!Exe { + const inputs = try allocator.alloc(Tensor, self.inputs.len); + defer allocator.free(inputs); + for (inputs, self.inputs) |*input, source| input.* = .fromShape(source.shape); + const opts: Compiler.Options = .{ .meshes = meshes, .program_name = self.program.name() }; + return self.program.vtable.compile(self.program.value, allocator, io, platform, opts, output, inputs); + } + }; +}; + +/// A program the loader compiles and runs to compute a tensor while loading +/// (`Binding.compute`): `forward(output, inputs)` of the program's value. +/// Model code makes one through `Source.bindProgram`, and reads it back +/// with `as`. +pub const Program = struct { + /// The program's value, in the store's arena. + value: *const anyopaque, + vtable: *const VTable, + + const VTable = struct { + name: []const u8, + compile: *const fn ( + value: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe, + eql: *const fn (a: *const anyopaque, b: *const anyopaque) bool, + }; + + /// `program`, its value copied into `arena`. Its type declares + /// `pub fn forward(self, output: Shape, inputs: []const Tensor) Tensor` + /// and is plain data, without pointers, so equal values are found by + /// value and compile once. + pub fn init(arena: std.mem.Allocator, program: anytype) std.mem.Allocator.Error!Program { + const T = @TypeOf(program); + comptime { + if (!isPlainData(T)) @compileError("TensorStore.Program: " ++ @typeName(T) ++ " must be plain data, without pointers"); + } + const copy = try arena.create(T); + copy.* = program; + return .{ .value = copy, .vtable = &Erased(T).vtable }; + } + + /// The program's value, if it is a `T`. + pub fn as(self: Program, comptime T: type) ?T { + if (self.vtable != &Erased(T).vtable) return null; + return Erased(T).typed(self.value); + } + + /// The program's type name, which names its compiled module in traces, + /// dumps and logs. + pub fn name(self: Program) []const u8 { + return self.vtable.name; + } + + /// Whether `a` and `b` are programs of one type with equal values. + fn eql(a: Program, b: Program) bool { + return a.vtable == b.vtable and a.vtable.eql(a.value, b.value); + } + + /// The vtable of programs of type `T`, one per type. + fn Erased(comptime T: type) type { + return struct { + const vtable: VTable = .{ .name = @typeName(T), .compile = compileValue, .eql = eqlValues }; + + fn typed(erased: *const anyopaque) T { + return @as(*const T, @ptrCast(@alignCast(erased))).*; + } + + fn compileValue( + erased: *const anyopaque, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + output: Shape, + inputs: []const Tensor, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io, platform, T.forward, .{ typed(erased), output, inputs }, opts); + } + + fn eqlValues(a: *const anyopaque, b: *const anyopaque) bool { + return std.meta.eql(typed(a), typed(b)); + } + }; + } + + /// Values compared by value: no pointer anywhere inside. + fn isPlainData(comptime T: type) bool { + return switch (@typeInfo(T)) { + .void, .bool, .int, .float, .@"enum" => true, + .array => |info| isPlainData(info.child), + .optional => |info| isPlainData(info.child), + .@"struct" => |info| inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + .@"union" => |info| info.tag_type != null and inline for (info.field_types) |Field| { + if (!isPlainData(Field)) break false; + } else true, + else => false, + }; + } +}; + +pub const Source = @import("Source.zig"); + +registry: safetensors.TensorRegistry, +meshes: []const *const Sharding.Mesh, +quantization: ?Quantization.Target, +id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), +allocator: std.mem.Allocator, +arena: std.heap.ArenaAllocator, + +pub const Options = struct { + /// The meshes `Source.withPartitioning` and `View.mesh` name. The slice + /// is copied; the meshes must outlive the store. + meshes: []const *const Sharding.Mesh = &.{}, + /// How the platform's FP8 GEMMs want block-FP8 weights, for the code + /// that binds them (`View.quantization`): `.of(platform)`. Required to + /// bind block-FP8 weights. + quantization: ?Quantization.Target = null, +}; + +/// A store over the entries of `registry`, which must be complete: add +/// every checkpoint first (`TensorRegistry.addRepo`). It takes the entries +/// and leaves `registry` empty. The store's map stays locked until +/// `deinit`, so a later insert panics in safe builds, and an entry's +/// address is its checkpoint tensor's identity (`View.get`). +pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); + const meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"); + var self: TensorStore = .{ + .registry = registry.*, + .meshes = meshes, + .quantization = opts.quantization, + .id_to_sources = .empty, + .allocator = allocator, + .arena = arena, + }; + registry.* = .init(registry.arena.child_allocator); + self.registry.tensors.lockPointers(); + return self; +} + +/// Frees the bindings, the sources and programs, then the registry entries +/// `init` took. No load started from the store may still run: `wait` or +/// `cancel` it first. +pub fn deinit(self: *TensorStore) void { + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); + self.registry.tensors.unlockPointers(); + self.registry.deinit(); +} + +/// The binding of tensor `id`, as the loader reads it; null when this store +/// did not bind it. +pub fn bindingOf(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); +} + +/// The root view, with no prefix. It points to this store, which must not +/// move afterwards. +pub fn view(self: *TensorStore) View { + return .{ .store = self }; +} + +/// The mesh `name` among `Options.meshes`; model code asks its `View`. +pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.io.TensorStore.init`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); +} + +/// What model code holds: a key prefix over the store's checkpoint, to look +/// tensors up, bind them and set their memory. A value that keeps its own +/// copy of the prefix (256 bytes at most, longer panics) and points to the +/// store. +pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + /// The store's root view, with no prefix. + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + /// This view under `prefix_` and a dot: `withPrefix("model")` then + /// `get("embed")` looks up `model.embed`. It only scopes lookups; + /// nothing is bound. + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// This view under `index` and a dot, as `withPrefix` of the number. + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + /// The keys' prefix under this view, with its trailing dot; "" at the + /// root. + pub fn prefix(self: *const View) []const u8 { + return self.prefix_buffer[0..self.prefix_length]; + } + + /// The checkpoint tensor `subkey`, the registry's own entry, whose + /// address is the tensor's identity. Null when the checkpoint has none. + pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); + return self.store.registry.tensors.getPtr(key); + } + + /// The checkpoint tensor `subkey`, with `tags` (null keeps the + /// checkpoint's), placed on `mesh_` by `partitioning` and bound: + /// `source(subkey, tags).withPartitioning(mesh_, partitioning).bind()`. + /// Panics when the checkpoint has no such tensor. + pub fn createTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `createTensor`, but null when the checkpoint has no such tensor. + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const source_ = self.maybeSource(subkey, tags) orelse return null; + return source_.withPartitioning(mesh_, partitioning).bind(); + } + + /// `createTensor` replicated on every device, without a mesh. + pub fn createReplicatedTensor(self: View, subkey: []const u8, comptime tags: anytype) Tensor { + return self.createTensor(subkey, tags, .replicated, .{}); + } + + /// Changes where the loader delivers `tensor` (`Memory`), for a + /// decision made after the tensor was bound. `tensor` must be bound in + /// this store, and not computed by a program: a program's output lands + /// in default device memory. + pub fn setMemory(self: View, tensor: Tensor, kind: Memory) void { + const binding = self.store.id_to_sources.getPtr(tensor.id) orelse + stdx.debug.panic("Tensor {} {f} has no binding in this store", .{ tensor.id, tensor.shape() }); + switch (binding.*) { + .load => |*load| load.memory = kind, + .compute => |compute| stdx.debug.panic("setMemory: program {s} computes tensor {} {f}, whose output lands in default device memory", .{ compute.program.name(), tensor.id, tensor.shape() }), + } + } + + /// The bytes on one device of every tensor the store delivers + /// `.host_mapped`, each counted by its shard: what XLA counts against + /// device memory for those views, which a model gives back through + /// `Compiler.Options.extra_device_memory_bytes`. The whole store's, + /// whatever this view's prefix; ask once every binding and `setMemory` + /// is done. A tensor whose mesh partitions do not divide its dimensions + /// is `IncompatibleSharding`, logged. + pub fn hostMappedBytes(self: View) !u64 { + var bytes: u64 = 0; + var it = self.store.id_to_sources.iterator(); + while (it.next()) |entry| switch (entry.value_ptr.*) { + .load => |load| if (load.memory == .host_mapped) { + const shard = Sharding.shardedShape(load.source.shape) catch |err| { + load_log.err("tensor {} {f}, delivered host_mapped: its dimensions are not divisible by its mesh partitions", .{ entry.key_ptr.*, load.source.shape }); + return err; + }; + bytes += shard.byteSize(); + }, + .compute => {}, + }; + return bytes; + } + + /// See `TensorStore.mesh`. + pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { + return self.store.mesh(name); + } + + /// See `Options.quantization`. Panics when the store has none: binding + /// block-FP8 weights requires it. + pub fn quantization(self: View) Quantization.Target { + return self.store.quantization orelse + stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); + } + + /// The checkpoint tensor `subkey` as a `Source`, with `tags` (null keeps + /// the checkpoint's). Panics when the checkpoint has no such tensor. + pub fn source(self: View, subkey: []const u8, comptime tags: anytype) Source { + return self.maybeSource(subkey, tags) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); + } + + /// Like `source`, but null when the checkpoint has no such tensor. + pub fn maybeSource(self: View, subkey: []const u8, comptime tags: anytype) ?Source { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + return .init(self, subkey, if (has_tags) parsed_tags.constSlice() else null); + } + + /// The checkpoint tensors under this view's prefix: a scan of every key. + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix(); + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } +}; + +test "get finds exact keys, the registry's own entries, and count the keys under a prefix" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "layer.experts.0.down_proj.weight", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + // The store took the entries. + try std.testing.expectEqual(0, registry.tensors.count()); + + const layer = store.view().withPrefix("layer"); + const entry = layer.get("experts.0.down_proj.weight").?; + try std.testing.expectEqual(entry, store.registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); + try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); + try std.testing.expectEqual(null, layer.get("weights")); + try std.testing.expectEqualStrings("layer.", layer.prefix()); + try std.testing.expectEqualStrings("", layer.root().prefix()); + try std.testing.expectEqualStrings("layer.experts.0.", layer.withPrefix("experts").withLayer(0).prefix()); + try std.testing.expectEqual(1, layer.withPrefix("experts").count()); + try std.testing.expectEqual(0, layer.withPrefix("gate").count()); + + // Every binding of the entry holds the entry itself. + const tensor = layer.createReplicatedTensor("experts.0.down_proj.weight", null); + try std.testing.expectEqual(entry, store.bindingOf(tensor.id).?.load.source.whole().?); +} + +test "programs of one type and equal values over the same placed shapes share an executable" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "t", .shape = .init(.{4}, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + const Scale = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const Shift = struct { + factor: u8, + + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + const shape: Shape = .init(.{4}, .u8); + const input = store.view().source("t", null).withPartitioning(.replicated, .{}); + const tensors = [_]Tensor{ + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), + Source.bindProgram(&.{input}, shape, Scale{ .factor = 2 }), + Source.bindProgram(&.{input}, shape, Shift{ .factor = 1 }), + }; + var computes: [tensors.len]Binding.Compute = undefined; + for (&computes, tensors) |*compute, tensor| compute.* = store.bindingOf(tensor.id).?.compute; + try std.testing.expect(computes[0].sameExecutable(shape, computes[1], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[2], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[3], shape)); + try std.testing.expect(!computes[0].sameExecutable(shape, computes[1], shape.withDtype(.i8))); + try std.testing.expectEqual(1, computes[3].program.as(Shift).?.factor); + try std.testing.expect(computes[3].program.as(Scale) == null); +} + +test "setMemory changes where the loader delivers a bound tensor" { + var registry: safetensors.TensorRegistry = .init(std.testing.allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "table", .shape = .init(.{ 4, 2 }, .u8), .offset = 0 }); + var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); + defer store.deinit(); + + const view_ = store.view(); + const tensor = view_.createReplicatedTensor("table", null); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .host_mapped); + try std.testing.expectEqual(Memory.host_mapped, store.bindingOf(tensor.id).?.load.memory); + view_.setMemory(tensor, .default); + try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); +} + +test "hostMappedBytes counts one device's shard of each host_mapped tensor" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .file_uri = "weights", .name = "rows", .shape = .init(.{ 8, 2 }, .u8), .offset = 0 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "scale", .shape = .init(.{4}, .u8), .offset = 16 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "dense", .shape = .init(.{4}, .u8), .offset = 20 }); + try registry.registerTensor(.{ .file_uri = "weights", .name = "odd", .shape = .init(.{ 3, 2 }, .u8), .offset = 24 }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + + const view_ = store.view(); + const rows = view_.createTensor("rows", .{ .row, .col }, .tp, .{ .row = .model }); + const scale = view_.createReplicatedTensor("scale", null); + _ = view_.createReplicatedTensor("dense", null); + try std.testing.expectEqual(0, try view_.hostMappedBytes()); + view_.setMemory(rows, .host_mapped); + view_.setMemory(scale, .host_mapped); + // Half the rows, and the whole replicated scale, from any prefix. + try std.testing.expectEqual(8 + 4, try view_.withPrefix("rows").hostMappedBytes()); + + // Three rows do not split over two devices. + const odd = view_.createTensor("odd", .{ .row, .col }, .tp, .{ .row = .model }); + view_.setMemory(odd, .host_mapped); + try std.testing.expectError(error.IncompatibleSharding, view_.hostMappedBytes()); +} diff --git a/zml/io/backend.zig b/zml/io/backend.zig new file mode 100644 index 00000000..f118f5fe --- /dev/null +++ b/zml/io/backend.zig @@ -0,0 +1,625 @@ +//! The seam between the front end (loader.zig) and its backends: the +//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), the +//! source request size (`requestSize`), and which backend and memory kinds +//! a platform takes (`loadsDirect`, `supportsMemory`). +const std = @import("std"); + +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Memory = @import("TensorStore.zig").Memory; +const constants = @import("../constants.zig"); +const buffered_loader = @import("buffered_loader.zig"); +const direct_loader = @import("direct_loader.zig"); +const dma_calibration = @import("dma_calibration.zig"); + +/// The largest source request, and so the largest DMA block: a request is +/// a whole number of blocks, at least one. +pub const max_read_request_size: usize = 32 * 1024 * 1024; + +/// The source request for DMA blocks of `block_size`: the read chunk +/// rounded up to whole blocks, one block at least. Null unless the block is +/// a nonzero multiple of `alignment` (`readAlignment`: reads land in +/// blocks), and the request is at most `max_read_request_size` and, with an +/// alignment, longer than two units, so a read widened by a unit at each +/// end fits. `zml.io.Loader.init` refuses options for which a block +/// calibration may select has none. +pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { + if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; + // Bounded before the rounding, which then cannot overflow. + const least = @max(read_chunk_size, block_size); + if (least > max_read_request_size) return null; + const request = std.mem.alignForwardAnyAlign(usize, least, block_size); + if (request > max_read_request_size) return null; + if (alignment != 0 and request <= 2 * alignment) return null; + return request; +} + +/// What the direct backend widens the reads of a file read directly to: +/// the VFS's alignment when `io` is a VFS and `direct_io` is not `off`, else +/// 0, every read exact. The VFS decides per local file whether it is read +/// directly (`VFS.useDirectIo`). +pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { + if (direct_io == .off or VFS.fromIo(io) == null) return 0; + return VFS.direct_io_alignment; +} + +/// `zml.io.Loader.Options` as `zml.io.Loader.init` checked them, the +/// checkpoint resolved to its VFS backend's read hints and the read width to +/// its default. See `zml.io.Loader.Options` for each field. +pub const Options = struct { + read_parallelism: usize, + read_hints: VFS.ReadHints = .local, + dma_block_size: ?usize = null, + direct_io: VFS.DirectIo = .auto, + /// The loader's checkpoint (loader.zig, `Loader.Options.checkpoint`) + /// when it is local and the VFS may read it direct (`readAlignment` is + /// not 0), else null; owned by the loader's set-up task. The direct + /// backend predicts from it how the checkpoint's files read, places its + /// arenas for that, and reads each of those files as predicted + /// (`direct_loader.Loader.create`); it keeps nothing of the path. + checkpoint: ?[]const u8 = null, + + /// The blocks calibration may select: `dma_block_size`, else + /// `dma_calibration.default_block_sizes`. + pub fn dmaBlockSizes(self: *const Options) []const usize { + return if (self.dma_block_size) |*block_size| block_size[0..1] else &dma_calibration.default_block_sizes; + } +}; + +/// One destination, its caller-owned output and the source ranges that fill +/// it, as the front end checked them (`checkParts`, `Backend.stridedRead`): +/// backends trust them. +/// Everything a spec names (its parts, their dimensions, the sources, the +/// output) stays alive until its await; backends borrow it. +pub const LoadSpec = struct { + /// The destination's global shape, which the output buffer keeps. Its + /// sharding places the bytes on the devices; a shape without a mesh is + /// replicated over every device (`Platform.placedShape`). + shape: Shape, + /// Where the bytes go, from the source's `TensorStore` binding. The + /// front end only submits kinds `supportsMemory` accepts, and a host + /// kind only with contiguous parts (`Lattice.dims` empty). + memory: Memory = .default, + /// Written whole once the destination's buffers exist, and not touched + /// before: a failure leaves it as it was or holding those buffers. + output: *Buffer, + /// What fills the destination's logical bytes, in ascending, disjoint + /// destination order; every byte no part covers is zero. + parts: []const LoadPart, +}; + +/// The bytes of `source` that `lattice` moves: its source offsets relative +/// to the tensor's data, its destination offsets into the destination's +/// logical bytes, the row-major bytes of its packed shape before the +/// sharding splits them. +pub const LoadPart = struct { + source: *const safetensors.Tensor, + lattice: Lattice, + + /// The whole of `source`, at the start of the destination. + pub fn whole(source: *const safetensors.Tensor) LoadPart { + return .{ .source = source, .lattice = .{ .width = source.byteSize() } }; + } +}; + +/// The most dimensions a part repeats over: `Source` gives each axis at +/// most two. +pub const max_part_dims = 2 * constants.MAX_RANK; + +/// A part's byte moves: the run of `width` bytes at `src_offset` and +/// `dst_offset`, and, for a strided part, one more for every index of +/// `dims` (outermost first), each offset moved by the index times its +/// strides. A zero source stride repeats the same bytes. Runs are numbered +/// row-major over `dims`. +pub const Lattice = struct { + src_offset: u64 = 0, + dst_offset: u64 = 0, + width: u64, + /// Empty for one contiguous run. + dims: []const Dim = &.{}, + + /// A step of the runs: `count` of them, each moving the source and + /// destination offsets by their strides. + pub const Dim = struct { + count: u64, + src_stride: u64, + dst_stride: u64, + }; + + pub const Run = struct { src: u64, dst: u64 }; + + pub fn runs(self: Lattice) u64 { + var n: u64 = 1; + for (self.dims) |d| n *= d.count; + return n; + } + + pub fn at(self: Lattice, index: u64) Run { + var rest = index; + var run: Run = .{ .src = self.src_offset, .dst = self.dst_offset }; + var d = self.dims.len; + while (d > 0) { + d -= 1; + const i = rest % self.dims[d].count; + rest /= self.dims[d].count; + run.src += i * self.dims[d].src_stride; + run.dst += i * self.dims[d].dst_stride; + } + return run; + } + + /// The source and destination bytes the runs span from their first + /// offsets, null when a count is zero or the bounds overflow. + pub fn spans(self: Lattice) ?struct { src: u64, dst: u64 } { + var src = self.width; + var dst = self.width; + for (self.dims) |d| { + if (d.count == 0) return null; + src = std.math.add(u64, std.math.mul(u64, d.count - 1, d.src_stride) catch return null, src) catch return null; + dst = std.math.add(u64, std.math.mul(u64, d.count - 1, d.dst_stride) catch return null, dst) catch return null; + } + return .{ .src = src, .dst = dst }; + } + + /// Whether each dimension steps past everything inside it on the + /// destination side: runs land in index order without overlapping. + /// Needs `spans` not null. + pub fn dstOrdered(self: Lattice) bool { + return ordered(self.dims, self.width, .dst, false); + } + + /// Whether the source runs come in index order without overlapping, + /// gaps allowed: the runs of an index range lie between the first's + /// start and the last's end. Needs `spans` not null. + pub fn srcOrdered(self: Lattice) bool { + return ordered(self.dims, self.width, .src, false); + } + + /// Whether the source runs are back to back in index order: an index + /// range reads one contiguous source range, each byte once. Needs + /// `spans` not null. + pub fn srcTiled(self: Lattice) bool { + return ordered(self.dims, self.width, .src, true); + } + + /// Whether each dimension steps past everything inside it on `side`, + /// exactly with `exact`. The extents it sums are bounded by `spans`. + fn ordered(dims: []const Dim, width: u64, comptime side: enum { src, dst }, exact: bool) bool { + var extent = width; + var d = dims.len; + while (d > 0) { + d -= 1; + if (dims[d].count < 2) continue; + const stride = if (side == .src) dims[d].src_stride else dims[d].dst_stride; + if (if (exact) stride != extent else stride < extent) return false; + extent += (dims[d].count - 1) * stride; + } + return true; + } + + /// The first run, in index order, that ends after destination offset + /// `offset`; `runs()` when none does. Needs `dstOrdered`. + pub fn firstEndingAfter(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst + self.width > offset) high = middle else low = middle + 1; + } + return low; + } + + /// The first run that starts at destination offset `offset` or later; + /// `runs()` when none does. Needs `dstOrdered`. + pub fn firstStartingFrom(self: Lattice, offset: u64) u64 { + var low: u64 = 0; + var high = self.runs(); + while (low < high) { + const middle = low + (high - low) / 2; + if (self.at(middle).dst >= offset) high = middle else low = middle + 1; + } + return low; + } + + /// The destination bytes of the runs inside `[start, end)`. + pub fn coveredIn(self: Lattice, start: u64, end: u64) u64 { + const first = self.firstEndingAfter(start); + const last = self.firstStartingFrom(end); + if (first >= last) return 0; + var covered = (last - first) * self.width; + const head = self.at(first).dst; + if (head < start) covered -= start - head; + const tail = self.at(last - 1).dst + self.width; + if (tail > end) covered -= tail - end; + return covered; + } + + /// Runs `first` to `end` (exclusive), in index order, without a + /// division per run. + pub const Cursor = struct { + lattice: Lattice, + index: u64, + end: u64, + counters: [max_part_dims]u64, + run: Run, + + pub fn init(lattice: Lattice, first: u64, end: u64) Cursor { + std.debug.assert(lattice.dims.len <= max_part_dims); + var cursor: Cursor = .{ .lattice = lattice, .index = first, .end = end, .counters = undefined, .run = lattice.at(first) }; + var rest = first; + var d = lattice.dims.len; + while (d > 0) { + d -= 1; + cursor.counters[d] = rest % lattice.dims[d].count; + rest /= lattice.dims[d].count; + } + return cursor; + } + + pub fn next(self: *Cursor) ?Run { + if (self.index >= self.end) return null; + const current = self.run; + self.index += 1; + if (self.index < self.end) { + const dims = self.lattice.dims; + var d = dims.len; + while (d > 0) { + d -= 1; + self.counters[d] += 1; + self.run.src +%= dims[d].src_stride; + self.run.dst +%= dims[d].dst_stride; + if (self.counters[d] < dims[d].count) break; + self.counters[d] = 0; + self.run.src -%= dims[d].count * dims[d].src_stride; + self.run.dst -%= dims[d].count * dims[d].dst_stride; + } + } + return current; + } + }; + + /// Copies runs `first` to `end` (exclusive): the `width` bytes at + /// `src + run.src` to `dst + run.dst`, `src` and `dst` being the + /// addresses the offsets count from (wrapping). Whole rows of the two + /// innermost dimensions go through nested loops, and a run of 1, 2, 4, + /// 8 or 16 bytes is one load and one store: a swizzle's 4-byte runs + /// cost a few cycles each instead of a cursor step and a `memcpy`. + pub fn copyRuns(self: Lattice, first: u64, end: u64, dst: usize, src: usize) void { + switch (self.width) { + inline 1, 2, 4, 8, 16 => |width| self.copyRunsOf(width, first, end, dst, src), + else => self.copyRunsOf(0, first, end, dst, src), + } + } + + /// `copyRuns` for runs of `width` bytes, or of `self.width` when zero. + fn copyRunsOf(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + const inner = self.dims[self.dims.len - @min(self.dims.len, 2) ..]; + var row: u64 = 1; + for (inner) |d| row *= d.count; + const rows_start = std.math.divCeil(u64, first, row) catch unreachable; + const rows_end = end / row; + if (rows_start >= rows_end) return self.copyEach(width, first, end, dst, src); + self.copyEach(width, first, rows_start * row, dst, src); + const outer: Lattice = .{ .src_offset = self.src_offset, .dst_offset = self.dst_offset, .width = self.width, .dims = self.dims[0 .. self.dims.len - inner.len] }; + var rows: Cursor = .init(outer, rows_start, rows_end); + while (rows.next()) |start| { + const row_dst = dst +% start.dst; + const row_src = src +% start.src; + switch (inner.len) { + 0 => copyRun(width, self.width, row_dst, row_src), + 1 => copyLine(width, self.width, inner[0], row_dst, row_src), + else => { + var line_dst = row_dst; + var line_src = row_src; + for (0..inner[0].count) |_| { + copyLine(width, self.width, inner[1], line_dst, line_src); + line_dst +%= inner[0].dst_stride; + line_src +%= inner[0].src_stride; + } + }, + } + } + self.copyEach(width, rows_end * row, end, dst, src); + } + + fn copyEach(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { + if (first >= end) return; + var cursor: Cursor = .init(self, first, end); + while (cursor.next()) |run| copyRun(width, self.width, dst +% run.dst, src +% run.src); + } + + inline fn copyLine(comptime width: usize, len: u64, d: Dim, dst: usize, src: usize) void { + var to = dst; + var from = src; + for (0..d.count) |_| { + copyRun(width, len, to, from); + to +%= d.dst_stride; + from +%= d.src_stride; + } + } + + inline fn copyRun(comptime width: usize, len: u64, dst: usize, src: usize) void { + if (width == 0) { + const n: usize = @intCast(len); + @memcpy(@as([*]u8, @ptrFromInt(dst))[0..n], @as([*]const u8, @ptrFromInt(src))[0..n]); + } else { + const Word = @Int(.unsigned, 8 * width); + @as(*align(1) Word, @ptrFromInt(dst)).* = @as(*align(1) const Word, @ptrFromInt(src)).*; + } + } +}; + +/// Whether `parts` keep the `LoadSpec` contract for a destination of +/// `destination_bytes`: there is at least one, each is non-empty and inside +/// its source and the destination, and they come in ascending destination +/// order, each starting at or after the previous contiguous part's end or +/// strided part's start. Each part's bytes end in their file at least +/// `VFS.direct_io_alignment` below `maxInt(u64)`, so a read of them widened +/// to the alignment does not overflow. Strided parts must land in index +/// order (`Lattice.dstOrdered`); interleaving ones are only checked against +/// each other by the bytes they cover together, since only `Source` makes +/// them, disjoint by construction. +pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { + if (parts.len == 0) return false; + var floor: u64 = 0; + var covered: u64 = 0; + for (parts) |part| { + const lattice = part.lattice; + if (lattice.width == 0 or lattice.dims.len > max_part_dims) return false; + const spans = lattice.spans() orelse return false; + if (!lattice.dstOrdered()) return false; + const source_end = std.math.add(u64, lattice.src_offset, spans.src) catch return false; + const file_end = std.math.add(u64, part.source.offset, source_end) catch return false; + const end = std.math.add(u64, lattice.dst_offset, spans.dst) catch return false; + if (source_end > part.source.byteSize() or file_end > std.math.maxInt(u64) - VFS.direct_io_alignment) return false; + if (end > destination_bytes or lattice.dst_offset < floor) return false; + floor = if (lattice.dims.len == 0) end else lattice.dst_offset; + covered +|= lattice.runs() * lattice.width; + } + return covered <= destination_bytes; +} + +/// Whether `platform` loads through the direct backend. Transfer-manager +/// support and host pinning are independent. CPU implements byte-range +/// transfers from ordinary pages but not DmaMap, so it shares coalescing +/// and bounded blocks without DMA support. The whole-tensor buffered path +/// has neither: on one B70, interleaved HF Qwen3.5-4B loads took 10.9 to +/// 16.3 s direct against about 48 s buffered. Furiosa stages in its +/// plugin's `.host_pinned` buffers (SDK dma-buf memory, which the SDK +/// transfers without pinning pages per request). TPU, neuron and metal +/// stay buffered until their transfer path is checked on hardware. +pub fn loadsDirect(platform: *const Platform) bool { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .cpu, .furiosa => true, + .tpu, .neuron, .metal => false, + }; +} + +/// Whether the loader can deliver into `kind` on `platform`. The host +/// kinds need the direct pipeline and a pinned host memory space on every +/// device. CPU loads through the direct backend, so it accepts them (its +/// plugin resolves every kind to its one memory space). +pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { + if (kind == .default) return true; + if (!loadsDirect(platform)) return false; + for (platform.devices) |*device| { + if (device.memory(.host_pinned) == null) return false; + } + return true; +} + +/// Bytes per device, by `platform.devices` index: `Platform.init` refuses +/// more devices than it holds. +pub const DeviceBytes = [Platform.MAX_NUM_DEVICES]u64; + +/// The transfer implementation selected for the platform (`loadsDirect`). +pub const Backend = union(enum) { + direct: *direct_loader.Loader, + buffered: *buffered_loader.Loader, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) !Backend { + return if (loadsDirect(platform)) + .{ .direct = try direct_loader.Loader.create(allocator, io, platform, opts) } + else + .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; + } + + /// Publishes `specs`, and completes one item of `progress` per spec. + /// Fails with what the backend detected in the files, host memory and + /// PJRT, logged there (a plain allocation failure is `OutOfMemory`, + /// unlogged). + pub fn submit(self: Backend, specs: []const LoadSpec, progress: std.Progress.Node) !Submission { + return switch (self) { + inline else => |loader, tag| @unionInit(Submission, @tagName(tag), try loader.submit(specs, progress)), + }; + } + + /// `read`, the source bytes every job of `part` reads whole, widened to + /// its file's alignment (0 when its jobs read only their own range: + /// source runs back to back, a contiguous part included), and `limit`, + /// the most one read takes: the front end refuses a part whose `read` + /// passes `limit`. Needs `checkParts`. The direct backend's limit is its + /// smallest read, whichever block calibration picks: the checkpoint's + /// read chunk (8 MiB locally) rounded up to whole blocks of the smallest + /// it may pick, 2 MiB or a pinned `dma_block_size` + /// (`direct_loader.Loader.stridedRead`); the buffered backend reads any + /// part. + pub fn stridedRead(self: Backend, part: LoadPart) StridedRead { + return switch (self) { + .direct => |loader| loader.stridedRead(part), + .buffered => .{ .read = 0, .limit = std.math.maxInt(u64) }, + }; + } + + pub const StridedRead = struct { read: u64, limit: u64 }; + + /// Waits for and retires `submission`, which this backend published. + /// Its batch is dangling afterwards. + pub fn await(self: Backend, submission: Submission) !void { + return switch (self) { + inline else => |loader, tag| loader.awaitBatch(@field(submission, @tagName(tag))), + }; + } + + /// Fails the backend unless it already failed: queued work is dropped, + /// work in flight drains, and every later await returns the first error. + pub fn abort(self: Backend) void { + switch (self) { + inline else => |loader| loader.abort(), + } + } + + pub fn destroy(self: Backend) void { + switch (self) { + inline else => |loader| loader.destroy(), + } + } +}; + +/// A published batch of the backend that published it. +pub const Submission = union(enum) { + direct: *direct_loader.Batch, + buffered: *buffered_loader.Batch, +}; + +/// A random lattice whose runs land in order (`Lattice.dstOrdered`), its +/// source strides anything, zero (a broadcast) included. +fn randomLattice(random: std.Random, dims: []Lattice.Dim) Lattice { + const width = random.intRangeAtMost(u64, 1, 17); + var extent = width; + var d = dims.len; + while (d > 0) { + d -= 1; + dims[d] = .{ + .count = random.intRangeAtMost(u64, 1, 5), + .src_stride = random.intRangeAtMost(u64, 0, 40), + .dst_stride = extent + random.intRangeAtMost(u64, 0, 3) * random.intRangeAtMost(u64, 0, 4), + }; + extent = dims[d].dst_stride * (dims[d].count - 1) + extent; + } + return .{ .src_offset = random.intRangeAtMost(u64, 0, 9), .dst_offset = random.intRangeAtMost(u64, 0, 9), .width = width, .dims = dims }; +} + +test "the cursor walks a range's runs where at places them, and copyRuns copies them" { + var prng: std.Random.DefaultPrng = .init(0x5eed); + const random = prng.random(); + var source: [4096]u8 = undefined; + for (&source, 0..) |*byte, i| byte.* = @truncate(i * 7 + 1); + var expected: [4096]u8 = undefined; + var actual: [4096]u8 = undefined; + var dims_buffer: [5]Lattice.Dim = undefined; + for (0..2000) |_| { + var lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); + // The widths `copyRuns` moves as one word, and others; a wider run + // may overlap its neighbours, which no part does. + if (random.boolean()) lattice.width = ([_]u64{ 1, 2, 4, 8, 16 })[random.uintLessThan(usize, 5)]; + const spans = lattice.spans().?; + if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len or !lattice.dstOrdered()) continue; + const runs = lattice.runs(); + const first = random.uintAtMost(u64, runs); + const end = random.intRangeAtMost(u64, first, runs); + @memset(&expected, 0); + @memset(&actual, 0); + var cursor: Lattice.Cursor = .init(lattice, first, end); + for (first..end) |index| { + const run = lattice.at(index); + try std.testing.expectEqual(run, cursor.next().?); + const width: usize = @intCast(lattice.width); + @memcpy(expected[@intCast(run.dst)..][0..width], source[@intCast(run.src)..][0..width]); + } + try std.testing.expectEqual(null, cursor.next()); + lattice.copyRuns(first, end, @intFromPtr(&actual), @intFromPtr(&source)); + try std.testing.expectEqualSlices(u8, &expected, &actual); + } +} + +test "checkParts refuses parts outside their source or destination, out of order or overlapping" { + // A source of 16 bytes into a destination of 32. + const source: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = 0 }; + // Sixteen bytes ending at the largest file offset a read widened to the + // alignment can reach, and one byte later. + const last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = std.math.maxInt(u64) - VFS.direct_io_alignment - 16 }; + const past_last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = last.offset + 1 }; + // Runs of 4 bytes, every other 4 bytes of the destination. + const every_other: []const Lattice.Dim = &.{.{ .count = 4, .src_stride = 4, .dst_stride = 8 }}; + const refused = [_][]const LoadPart{ + &.{}, + &.{.{ .source = &source, .lattice = .{ .width = 0 } }}, + // Past the source, past the destination, an overflow, and past the + // largest file offset. + &.{.{ .source = &source, .lattice = .{ .src_offset = 8, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 24, .width = 9 } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = std.math.maxInt(u64), .width = 1 } }}, + &.{.whole(&past_last)}, + // Overlapping, and descending. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } } }, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4 } }, .{ .source = &source, .lattice = .{ .width = 4 } } }, + // Strided: runs that overlap on the destination, a dimension of no + // step, strides whose span overflows, past the source, past the + // destination, and descending. + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 2 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 0, .src_stride = 4, .dst_stride = 8 }} } }}, + &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 1 << 40, .src_stride = 0, .dst_stride = 1 << 40 }} } }}, + &.{.{ .source = &source, .lattice = .{ .src_offset = 4, .width = 4, .dims = every_other } }}, + &.{.{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = every_other } }}, + &.{ .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } } }, + // Interleaving strided parts that cover more than the destination, + // and contiguous parts overlapping beside a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + }; + for (refused) |parts| try std.testing.expect(!checkParts(parts, 32)); + const accepted = [_][]const LoadPart{ + // Back to back. + &.{ .{ .source = &source, .lattice = .{ .width = 16 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 16 } } }, + // Two strided parts interleaving their runs. + &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, + // A contiguous part, then a strided one. + &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, + // A broadcast: one source run repeated over the destination. + &.{.{ .source = &source, .lattice = .{ .width = 8, .dims = &.{.{ .count = 4, .src_stride = 0, .dst_stride = 8 }} } }}, + // Ending at the largest file offset. + &.{.whole(&last)}, + }; + for (accepted) |parts| try std.testing.expect(checkParts(parts, 32)); +} + +test "requestSize rounds the read chunk up to whole blocks of whole alignment units" { + const mib: usize = 1 << 20; + // One block at least, the chunk rounded up to whole blocks, and a + // rounded request past the largest refused. + try std.testing.expectEqual(16 * mib, requestSize(8 * mib, 16 * mib, 4096)); + try std.testing.expectEqual(9 * mib, requestSize(8 * mib, 3 * mib, 4096)); + try std.testing.expectEqual(null, requestSize(32 * mib, 24 * mib, 0)); + // Without an alignment any nonzero block; with one, only its multiples. + try std.testing.expectEqual(8 * mib + 2048, requestSize(8 * mib, 8 * mib + 2048, 0)); + try std.testing.expectEqual(null, requestSize(8 * mib, 8 * mib + 2048, 4096)); + try std.testing.expectEqual(null, requestSize(8 * mib, 0, 0)); + // A request of two alignment units cannot hold a read widened at both + // ends. + try std.testing.expectEqual(null, requestSize(4096, 8192, 4096)); + try std.testing.expectEqual(3 * 4096, requestSize(4096, 3 * 4096, 4096)); +} + +test "the options pin a DMA block or leave calibration its candidates, and only a VFS widens reads" { + const pinned: Options = .{ .read_parallelism = 1, .dma_block_size = 4 << 20 }; + try std.testing.expectEqualSlices(usize, &.{4 << 20}, pinned.dmaBlockSizes()); + const calibrated: Options = .{ .read_parallelism = 1 }; + try std.testing.expectEqualSlices(usize, &dma_calibration.default_block_sizes, calibrated.dmaBlockSizes()); + + var filesystem = try VFS.init(std.testing.allocator, std.testing.io); + defer filesystem.deinit(); + try std.testing.expectEqual(VFS.direct_io_alignment, readAlignment(filesystem.io(), .auto)); + try std.testing.expectEqual(0, readAlignment(filesystem.io(), .off)); + try std.testing.expectEqual(0, readAlignment(std.testing.io, .on)); +} diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig new file mode 100644 index 00000000..96cfac3e --- /dev/null +++ b/zml/io/buffered_loader.zig @@ -0,0 +1,839 @@ +//! Whole-tensor staging for platforms that use Buffer.from: TPU, neuron +//! and metal, until the direct path is checked on them. A composite +//! destination is assembled in the staging: its gaps zeroed, each +//! part read at its offset (a strided part reads its source range once and +//! copies each run where it lands), then the whole tensor is placed as any +//! other. Each batch runs up to the read width of workers over its specs; +//! read permits bound the source reads and one byte budget the host +//! staging, live and kept for reuse. There is no calibration and no pinned +//! memory. The shared front end owns source lookup, programs and admission. +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const backend = @import("backend.zig"); +const testing = @import("../testing.zig"); +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const log = @import("log.zig").load; + +/// The host staging budget in largest requests: this many times the +/// largest staging request submitted (`stagingBytes`), for the buffers in +/// use and those kept for reuse alike. Twelve rather than the read width: +/// at a high-latency width of 32 the read width would nearly triple the +/// host staging for nothing this path can use (on such a source a tensor +/// splits its read across the free permits). +const staging_tensors: usize = 12; + +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// Workers per batch, so the threads grow with the batches in flight: + /// the read permits bound their reads and the staging budget their + /// bytes, not their number. + width: usize, + /// The smallest claim of a tensor's read split across helpers + /// (`ChunkedRead.claim`). A tensor is one positional read unless spare + /// permits split it on a high-latency source. + read_chunk_size: usize, + /// The widest fan-out one tensor may take. One on a source whose reads + /// are bandwidth-bound rather than round-trip bound: splitting there only + /// spends tasks, and measurably so -- a replicated local load, whose long + /// transfers keep permits free, loses time to helpers it has no use for. + tensor_workers: usize, + permits: ReadPermits, + staging: Staging, + first_error: stdx.FirstError(anyerror) = .{}, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: backend.Options, + ) !*Loader { + const self = try allocator.create(Loader); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .width = opts.read_parallelism, + .staging = .{ .slots = @min(opts.read_parallelism, staging_tensors) }, + .read_chunk_size = opts.read_hints.read_chunk_size, + // A high-latency source may use spare read permits to split a + // tensor already staging: more concurrency without more host memory. + .tensor_workers = if (opts.read_hints.high_latency) opts.read_parallelism else 1, + .permits = .{ .free = opts.read_parallelism }, + }; + return self; + } + + /// Starts up to `width` workers over the specs, each assembling one + /// destination at a time from its parts. The batch borrows `specs` and + /// everything they name until it is awaited (`backend.LoadSpec`). On an + /// io that starts no worker, the batch loads inside this call. Nothing + /// runs when this fails. + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); + var largest: usize = 0; + for (specs) |spec| largest = @max(largest, stagingBytes(spec)); + const batch = try self.allocator.create(Batch); + batch.* = .{ .specs = specs, .progress = progress }; + self.staging.widen(self.io, largest); + // `concurrent`, never `async`: a busy Threaded io runs `async` + // inline, and an inline first worker would drain the whole cursor + // here. + var started: usize = 0; + for (0..@min(self.width, specs.len)) |_| { + batch.workers.concurrent(self.io, work, .{ self, batch }) catch break; + started += 1; + } + if (started == 0) self.work(batch); + return batch; + } + + /// Waits for the batch's workers, frees it and returns the sticky error. + /// Uncancelable: the workers borrow the batch and its specs until they + /// return. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + const protection = self.io.swapCancelProtection(.blocked); + batch.workers.await(self.io) catch |err| switch (err) { + error.Canceled => unreachable, + }; + _ = self.io.swapCancelProtection(protection); + self.allocator.destroy(batch); + try self.checkOpen(); + } + + /// Fails the loader unless it already failed: workers start no other + /// tensor, and one waiting for its staging returns once it gets it. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// Every batch was awaited, so no worker runs. + pub fn destroy(self: *Loader) void { + self.staging.deinit(self.allocator); + self.allocator.destroy(self); + } + + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) log.failure(err, fmt, args); + } + + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + + /// One of the batch's workers: loads the next unclaimed spec until none + /// is left. + fn work(self: *Loader, batch: *Batch) void { + while (true) { + const index = batch.next.fetchAdd(1, .monotonic); + if (index >= batch.specs.len) return; + self.loadOne(batch.specs[index]); + // One item on the caller's node: a child node per tensor runs + // out of std.Progress's 127. + batch.progress.completeOne(); + } + } + + fn loadOne(self: *Loader, spec: LoadSpec) void { + if (self.first_error.get() != null) return; + std.debug.assert(spec.parts.len != 0); + const tensor_bytes = spec.shape.byteSize(); + const staging = self.staging.acquire(self.allocator, self.io, stagingBytes(spec)) catch |err| + return self.fail(err, "allocate staging for", spec, &spec.parts[0]); + defer self.staging.release(self.allocator, self.io, staging); + // The staging may have taken a while: a failure meanwhile leaves + // this tensor unstarted. + if (self.first_error.get() != null) return; + const tensor = staging[0..tensor_bytes]; + // Staging is reused, so a byte no part writes would publish another + // tensor's bytes rather than zero: the gaps are zeroed first. + var strided = false; + for (spec.parts) |part| strided = strided or part.lattice.dims.len != 0; + if (strided) { + @memset(tensor, 0); + } else { + // Only the gaps: the parts overwrite the rest. + var covered_end: usize = 0; + for (spec.parts) |part| { + const start: usize = @intCast(part.lattice.dst_offset); + @memset(tensor[covered_end..start], 0); + covered_end = start + @as(usize, @intCast(part.lattice.width)); + } + @memset(tensor[covered_end..], 0); + } + for (spec.parts) |*part| { + const file = std.Io.Dir.openFile(.cwd(), self.io, part.source.file_uri, .{ .mode = .read_only }) catch |err| + return self.fail(err, "open source for", spec, part); + defer file.close(self.io); + const lattice = part.lattice; + if (lattice.dims.len == 0) { + const start: usize = @intCast(lattice.dst_offset); + self.readInto(part.source, file, lattice.src_offset, tensor[start..][0..@intCast(lattice.width)]) catch |err| + return self.fail(err, "read", spec, part); + continue; + } + // A strided part: its source range once, into the staging past + // the tensor, then each run where it lands. + const source = staging[tensor_bytes..][0..@intCast(lattice.spans().?.src)]; + self.readInto(part.source, file, lattice.src_offset, source) catch |err| + return self.fail(err, "read", spec, part); + lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); + } + // The front end asks no host kind of this backend + // (`backend.supportsMemory`). + std.debug.assert(spec.memory == .default); + // The shape's sharding places the bytes; a shape without a mesh is + // replicated, as `Buffer.from` places it. + spec.output.* = Buffer.from(self.io, self.platform, spec.shape, tensor, .{ .wait = true }) catch |err| + return self.fail(err, "transfer", spec, &spec.parts[0]); + } + + /// Fails the loader with `err`, which `step` of `spec` met at `part`. + fn fail(self: *Loader, err: anyerror, step: []const u8, spec: LoadSpec, part: *const LoadPart) void { + self.recordError(err, "{s} tensor {s}: file={s}, offset={d}, source_bytes={d}, parts={d}, destination_bytes={d}", .{ step, part.source.name, part.source.file_uri, part.source.offset, part.source.byteSize(), spec.parts.len, spec.shape.byteSize() }); + } + + /// Fills `destination` from `source`'s bytes at `source_offset`, read + /// from `file`, splitting it across helper tasks when the source permits + /// are free. The permits are released before the caller's transfer, so a + /// tensor staging to the device lends its read budget to one still + /// reading. + fn readInto( + self: *Loader, + source: *const safetensors.Tensor, + file: std.Io.File, + source_offset: u64, + destination: []u8, + ) safetensors.Tensor.RangeError!void { + var chunked: ChunkedRead = .{ + .io = self.io, + .source = source, + .file = file, + .source_offset = source_offset, + .destination = destination, + .chunk_size = self.read_chunk_size, + }; + // Splitting below the read chunk buys nothing, so that is the + // most workers this tensor can use. + var useful_workers = @min( + self.tensor_workers, + destination.len / self.read_chunk_size + + @intFromBool(destination.len % self.read_chunk_size != 0), + ); + + self.permits.acquire(self.io); + var held: usize = 1; + var group: std.Io.Group = .init; + while (true) { + // Helpers only ever take permits that are already free, so a busy + // source sees one read per tensor. The check repeats between + // claims because the tensors that finish first hand their budget + // to the one still reading -- which is the long tensor that + // needed the help. + // Nothing unclaimed means a helper would only start and stop. + if (held < useful_workers and chunked.next.load(.monotonic) < destination.len) { + const extra = self.permits.tryAcquire(self.io, useful_workers - held); + if (extra != 0) { + // Claims size themselves by the width, so it is raised + // before a helper can claim. `concurrent`, since a busy + // io runs `async` inline, on this task. + chunked.workers.store(held + extra, .monotonic); + var started: usize = 0; + while (started < extra) : (started += 1) { + group.concurrent(self.io, ChunkedRead.run, .{&chunked}) catch break; + } + held += started; + if (started < extra) { + // The io starts no more: the rest of the permits go + // back, and this tensor stops asking. + self.permits.release(self.io, extra - started); + chunked.workers.store(held, .monotonic); + useful_workers = held; + } + } + } + if (!chunked.readOne()) break; + } + group.await(self.io) catch {}; + self.permits.release(self.io, held); + + if (chunked.failure.get()) |err| return err; + } +}; + +/// The staging one spec takes: its tensor, then room for its widest strided +/// part's source range, which is read once before its runs are copied out. +fn stagingBytes(spec: LoadSpec) usize { + var span: usize = 0; + for (spec.parts) |part| { + if (part.lattice.dims.len != 0) span = @max(span, @as(usize, @intCast(part.lattice.spans().?.src))); + } + return spec.shape.byteSize() + span; +} + +/// One buffered submission: its workers claim specs from `next`. +pub const Batch = struct { + specs: []const LoadSpec, + progress: std.Progress.Node, + next: std.atomic.Value(usize) = .init(0), + workers: std.Io.Group = .init, +}; + +/// Source reads in flight across every task of the buffered backend, at most +/// `read_parallelism`. On a high-latency source, a tensor wider than the +/// hints' `read_chunk_size` may split its read across the permits that are +/// free, for parallel streams. +const ReadPermits = struct { + mutex: std.Io.Mutex = .init, + available: std.Io.Condition = .init, + free: usize, + + fn acquire(self: *ReadPermits, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free == 0) self.available.waitUncancelable(io, &self.mutex); + self.free -= 1; + } + + /// Up to `count` permits, without waiting: how many were free is how much + /// concurrency this tensor gets, so a loaded source is left alone. + fn tryAcquire(self: *ReadPermits, io: std.Io, count: usize) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const taken = @min(count, self.free); + self.free -= taken; + return taken; + } + + fn release(self: *ReadPermits, io: std.Io, count: usize) void { + if (count == 0) return; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.free += count; + for (0..count) |_| self.available.signal(io); + } +}; + +/// One tensor's read, claimed from a shared byte cursor by the owning task +/// and its helpers. Positional reads leave the file as it was, so they can +/// run concurrently on one file. +/// +/// A claim is the tensor split `workers` ways, and never smaller than the +/// hints' `read_chunk_size` -- that field is a minimum request size, and +/// on a high-latency source a request costs a round trip whether or not it +/// runs beside another. So one worker takes the whole tensor in a single +/// read, and only the concurrency the source can actually carry turns into +/// extra requests. +const ChunkedRead = struct { + io: std.Io, + source: *const safetensors.Tensor, + file: std.Io.File, + /// Where `destination` starts in the tensor's bytes. + source_offset: u64 = 0, + destination: []u8, + chunk_size: usize, + /// Reads this tensor may have in flight: the owner raises it before it + /// starts helpers on newly taken permits, and lowers it to the helpers + /// that started. + workers: std.atomic.Value(usize) = .init(1), + /// Bytes already claimed. + next: std.atomic.Value(usize) = .init(0), + failure: stdx.FirstError(safetensors.Tensor.RangeError) = .{}, + + fn run(self: *ChunkedRead) void { + while (self.readOne()) {} + } + + /// Reads the next unclaimed span. False once the tensor is fully claimed, + /// or once a read failed. + fn readOne(self: *ChunkedRead) bool { + if (self.failure.get() != null) return false; + const span = self.claim() orelse return false; + self.source.readRange(self.io, self.file, span.bytes, self.source_offset + span.offset) catch |err| { + _ = self.failure.set(err); + return false; + }; + return true; + } + + fn claim(self: *ChunkedRead) ?struct { offset: usize, bytes: []u8 } { + var start = self.next.load(.monotonic); + while (start < self.destination.len) { + const workers = self.workers.load(.monotonic); + const share = self.destination.len / workers + + @intFromBool(self.destination.len % workers != 0); + const len = @min(@max(self.chunk_size, share), self.destination.len - start); + if (self.next.cmpxchgWeak(start, start + len, .monotonic, .monotonic)) |actual| { + start = actual; + continue; + } + return .{ .offset = start, .bytes = self.destination[start..][0..len] }; + } + return null; + } +}; + +/// Host staging for the buffered backend: one budget for the buffers the +/// running tensors hold and those kept for reuse, which are handed back on +/// completion instead of freed. Mapping, faulting and then unmapping a whole +/// tensor's worth of pages for every tensor takes half of a serial buffered +/// load of a 14.96 GiB checkpoint, and a fifth of a concurrent one. +/// +/// The budget is `slots` times the largest request submitted, so no +/// request is wider than it. A request takes the smallest kept buffer that +/// fits, which counts in full while it is held: small tensors run far wider +/// than the slot count on small buffers, but once only large ones are kept +/// they take those, and run about `slots` wide. +const Staging = struct { + mutex: std.Io.Mutex = .init, + room: std.Io.Condition = .init, + /// The budget in largest requests (`staging_tensors`). + slots: usize, + /// Zero until the first submission reports its largest request. + budget: usize = 0, + /// What the running tensors hold. + live_bytes: usize = 0, + /// Buffers no tensor holds, and their total length. + free: std.ArrayList([]u8) = .empty, + retained_bytes: usize = 0, + + /// Raises the budget to `slots` of `largest`. Submissions only ever widen + /// it: a later submission of smaller tensors cannot shrink what is + /// already running. + fn widen(self: *Staging, io: std.Io, largest: usize) void { + const wanted = largest *| self.slots; + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (wanted <= self.budget) return; + self.budget = wanted; + self.room.broadcast(io); + } + + /// A whole allocation of at least `size` bytes, with undefined contents: + /// the allocation itself, not a slice of one. The caller slices it for + /// its own use and returns this exact slice to `release`, whatever + /// happens. Waits until a retained buffer fits, or `size` more bytes + /// fit the budget beside the live ones. `size` is at most the budget: + /// `widen` ran for the request's submission. + fn acquire(self: *Staging, allocator: std.mem.Allocator, io: std.Io, size: usize) ![]u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(size <= self.budget); + while (true) { + if (self.take(size)) |buffer| return buffer; + if (self.live_bytes + size <= self.budget) break; + self.room.waitUncancelable(io, &self.mutex); + } + // No retained buffer fits, so each is smaller than this request: + // free them, smallest first, until it fits beside the rest. + while (self.live_bytes + self.retained_bytes + size > self.budget) { + const dropped = self.free.swapRemove(self.smallest()); + self.retained_bytes -= dropped.len; + allocator.free(dropped); + } + const fresh = try allocator.alloc(u8, size); + self.live_bytes += fresh.len; + std.debug.assert(self.live_bytes + self.retained_bytes <= self.budget); + return fresh; + } + + /// `buffer` must be what `acquire` returned, not a slice of it. It is + /// kept: the budget already counts it. + fn release(self: *Staging, allocator: std.mem.Allocator, io: std.Io, buffer: []u8) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.live_bytes -= buffer.len; + self.room.broadcast(io); + self.free.append(allocator, buffer) catch return allocator.free(buffer); + self.retained_bytes += buffer.len; + } + + /// Every tensor is done, so every buffer is back. + fn deinit(self: *Staging, allocator: std.mem.Allocator) void { + std.debug.assert(self.live_bytes == 0); + for (self.free.items) |buffer| allocator.free(buffer); + self.free.deinit(allocator); + } + + /// The smallest retained buffer that fits, now live, so the big ones + /// stay free for the big tensors. Called with `mutex` held. + fn take(self: *Staging, size: usize) ?[]u8 { + var best: ?usize = null; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < size) continue; + if (best == null or buffer.len < self.free.items[best.?].len) best = i; + } + const buffer = self.free.swapRemove(best orelse return null); + self.retained_bytes -= buffer.len; + self.live_bytes += buffer.len; + return buffer; + } + + /// Called with `mutex` held and a buffer retained. + fn smallest(self: *const Staging) usize { + var index: usize = 0; + for (self.free.items, 0..) |buffer, i| { + if (buffer.len < self.free.items[index].len) index = i; + } + return index; + } +}; + +test "chunked read splits a tensor only as wide as it has workers" { + var destination: [1000]u8 = undefined; + var work: ChunkedRead = .{ + .io = undefined, + .source = undefined, + .file = undefined, + .destination = &destination, + .chunk_size = 100, + }; + + // One worker reads the whole tensor, exactly as an unsplit read would. + const whole = work.claim().?; + try std.testing.expectEqual(@as(usize, 0), whole.offset); + try std.testing.expectEqual(@as(usize, 1000), whole.bytes.len); + try std.testing.expect(work.claim() == null); + + // Four workers take a quarter each. + work.next = .init(0); + work.workers = .init(4); + for (0..4) |i| { + const span = work.claim().?; + try std.testing.expectEqual(i * 250, span.offset); + try std.testing.expectEqual(@as(usize, 250), span.bytes.len); + } + try std.testing.expect(work.claim() == null); + + // Never below the read chunk, however many workers there are. + work.next = .init(0); + work.workers = .init(50); + const floored = work.claim().?; + try std.testing.expectEqual(@as(usize, 100), floored.bytes.len); +} + +test "staging holds live and retained bytes within one budget" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var staging: Staging = .{ .slots = 3 }; + defer staging.deinit(allocator); + + // Three slots of the largest request is the budget, and a later, + // smaller submission cannot shrink it. + staging.widen(io, 16); + staging.widen(io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.budget); + + // With nothing kept, small requests run far wider than the slot count, + // and their buffers are kept. + var small: [12][]u8 = undefined; + for (&small) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (small) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A request no retained buffer fits frees as many as it needs room + // from. + const large = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 32), staging.retained_bytes); + staging.release(allocator, io, large); + // The smallest buffer that fits comes back to a smaller request, so the + // 16 stays for one the 4s cannot serve. The first take moves the 16 to + // the front of the list, ahead of the second's 4. + var reused: [2][]u8 = undefined; + for (&reused) |*buffer| { + buffer.* = try staging.acquire(allocator, io, 3); + try std.testing.expectEqual(@as(usize, 4), buffer.len); + } + try std.testing.expectEqual(@as(usize, 40), staging.retained_bytes); + for (reused) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); + + // A full budget holds the next request until a buffer comes back, then + // hands it that buffer. + var held: [3][]u8 = undefined; + for (&held) |*buffer| buffer.* = try staging.acquire(allocator, io, 16); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes + staging.retained_bytes); + var waiting = try io.concurrent(Staging.acquire, .{ &staging, allocator, io, 16 }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + staging.release(allocator, io, held[0]); + held[0] = try waiting.await(io); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (held) |buffer| staging.release(allocator, io, buffer); + + // Once only large buffers are kept, small requests take them, each + // counted in full: three fill the budget. + var on_large: [3][]u8 = undefined; + for (&on_large) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); + try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); + for (on_large) |buffer| staging.release(allocator, io, buffer); + try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); +} + +test "buffered loader assembles a composite destination in reused staging" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 2); + defer devices.deinit(); + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var a: [48]u8 = undefined; + testing.pattern(&a, 1); + var b: [32]u8 = undefined; + testing.pattern(&b, 2); + var full: [128]u8 = undefined; + testing.pattern(&full, 3); + var big: [256]u8 = undefined; + testing.pattern(&big, 4); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &b }, + .{ .name = "full", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &full }, + .{ .name = "big", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &big }, + }); + // Rows split over the two devices: b at row 1, a's last two rows (from + // byte 16 of a) at row 4, and a strided part, the left halves of full's + // rows 0 and 2, at rows 6 and 7, around a leading row, a gap and the + // right halves of zeros. + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const columns = [_]backend.Lattice.Dim{.{ .count = 2, .src_stride = 32, .dst_stride = 16 }}; + const parts = [_]LoadPart{ + .{ .source = registry.tensors.getPtr("b").?, .lattice = .{ .dst_offset = 16, .width = 32 } }, + .{ .source = registry.tensors.getPtr("a").?, .lattice = .{ .src_offset = 16, .dst_offset = 64, .width = 32 } }, + .{ .source = registry.tensors.getPtr("full").?, .lattice = .{ .dst_offset = 96, .width = 8, .dims = &columns } }, + }; + try std.testing.expect(backend.checkParts(&parts, shape.byteSize())); + var expected: [128]u8 = @splat(0); + @memcpy(expected[16..48], &b); + @memcpy(expected[64..96], a[16..48]); + @memcpy(expected[96..104], full[0..8]); + @memcpy(expected[112..120], full[32..40]); + const whole = [_]LoadPart{.whole(registry.tensors.getPtr("full").?)}; + const big_shape = Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + const big_whole = [_]LoadPart{.whole(registry.tensors.getPtr("big").?)}; + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + // A bigger tensor first, so the composite, its strided source range + // included, reuses its staging buffer and only the zeroing keeps its + // bytes out of the gaps. + var first: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = big_shape, .output = &first, .parts = &big_whole }}, .none)); + defer first.deinit(); + try testing.expectShards(allocator, io, &first, &big, .default); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &expected, .default); + // One buffer was ever allocated, within the budget of two slots of the + // largest request. + try std.testing.expectEqual(@as(usize, 512), loader.staging.budget); + try std.testing.expectEqual(@as(usize, 256), loader.staging.retained_bytes); + + // A source whose file does not exist fails the worker, and the error + // is sticky from then on. + var missing = registry.tensors.get("full").?; + missing.file_uri = "missing.safetensors"; + const missing_parts = [_]LoadPart{.whole(&missing)}; + var never_written: Buffer = undefined; + const batch = try loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &missing_parts }}, .none); + try std.testing.expectError(error.FileNotFound, loader.awaitBatch(batch)); + // `expectError` would print the batch, and every type its specs name. + if (loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &whole }}, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.FileNotFound, err); +} + +test "a worker waiting for its staging loads nothing once the loader failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + // The test holds the whole budget, one slot of the tensor, so the + // worker waits for its staging until the loader failed. + loader.staging.widen(io, bytes.len); + const held = try loader.staging.acquire(allocator, io, bytes.len); + var output: Buffer = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + defer output.deinit(); + const batch = try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none); + try io.sleep(.fromMilliseconds(5), .awake); + loader.abort(); + loader.staging.release(allocator, io, held); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + try std.testing.expectEqual(@as(usize, 0), output._shards.len); +} + +test "an aborted loader leaves each output as it was or whole, and stays failed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [64]u8 = undefined; + testing.pattern(&bytes, 1); + try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, + }); + const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); + const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; + // One worker over many specs, so the abort lands among them. + var specs: [64]LoadSpec = undefined; + var outputs: [64]Buffer = undefined; + for (&specs, &outputs) |*spec, *output| { + output.* = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; + spec.* = .{ .shape = shape, .output = output, .parts = &parts }; + } + defer for (&outputs) |*output| output.deinit(); + + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + loader.abort(); + try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); + for (&outputs) |*output| { + if (output._shards.len != 0) try testing.expectShards(allocator, io, output, &bytes, .default); + } + if (loader.submit(&specs, .none)) |_| + return error.TestUnexpectedResult + else |err| + try std.testing.expectEqual(error.Canceled, err); +} + +test "buffered loader runs overlapping batches over their cursors, with or without concurrency" { + const allocator = std.testing.allocator; + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // Tensors of 2 to 32 rows of 8 u16, so the staging budget holds a + // varying number of them. + const count = 24; + var bytes: [count][512]u8 = undefined; + var names: [count][8]u8 = undefined; + var dims: [count][2]i64 = undefined; + var entries: [count]testing.TensorBytes = undefined; + for (&bytes, &names, &dims, &entries, 0..) |*data, *name, *dim, *entry, i| { + const rows = 2 * (i % 16 + 1); + testing.pattern(data[0 .. rows * 16], @intCast(i + 1)); + dim.* = .{ @intCast(rows), 8 }; + entry.* = .{ .name = try std.fmt.bufPrint(name, "t{d}", .{i}), .dtype = "U16", .dims = dim, .bytes = data[0 .. rows * 16] }; + } + try testing.writeSafetensors(allocator, std.testing.io, tmp.dir, "weights.safetensors", ®istry, &entries); + + var parts: [count][1]LoadPart = undefined; + var specs: [count]LoadSpec = undefined; + var outputs: [count]Buffer = undefined; + for (&parts, &specs, &outputs, entries) |*part, *spec, *output, entry| { + part[0] = .whole(registry.tensors.getPtr(entry.name).?); + const shape = Shape.init(.{ .row = entry.dims[0], .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); + spec.* = .{ .shape = shape, .output = output, .parts = part }; + } + + var no_concurrency: std.Io.Threaded = .init(allocator, .{ .async_limit = .nothing, .concurrent_limit = .nothing }); + defer no_concurrency.deinit(); + for ([_]std.Io{ std.testing.io, no_concurrency.io() }) |io| { + // A high-latency source with 32-byte chunks: helpers split each + // tensor whenever a permit is free. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = 32, .high_latency = true }, + }); + defer loader.destroy(); + // Both batches are in flight before either is awaited, each with + // its own workers: the permits bound their reads and the staging + // budget their bytes (`Staging.acquire` asserts it). + const first = try loader.submit(specs[0 .. count / 2], .none); + const second = try loader.submit(specs[count / 2 ..], .none); + try loader.awaitBatch(first); + try loader.awaitBatch(second); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); + try std.testing.expect(loader.staging.retained_bytes <= loader.staging.budget); + try std.testing.expectEqual(@as(usize, 0), loader.staging.live_bytes); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); + } +} + +test "a tensor from a high-latency HTTP source splits its read over the free permits" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + + var object: [4096]u8 = undefined; + testing.pattern(&object, 1); + var server = try VFS.MockServer.init(io, &object, .{}); + defer server.deinit(io); + try server.start(io); + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + + var url_buffer: [64]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + const source: safetensors.Tensor = .{ .file_uri = url, .name = "x", .shape = .init(.{object.len}, .u8), .offset = 0 }; + const parts = [_]LoadPart{.whole(&source)}; + + // Four free permits and a quarter of the tensor as the read chunk: the + // worker and three helpers each read a quarter, one range GET each. + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ + .read_parallelism = 4, + .read_hints = .{ .read_chunk_size = object.len / 4, .high_latency = true }, + }); + defer loader.destroy(); + var output: Buffer = undefined; + try loader.awaitBatch(try loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none)); + defer output.deinit(); + try server.stop(io); + try testing.expectShards(allocator, io, &output, &object, .default); + try std.testing.expectEqual(@as(usize, 4), server.get_requests.load(.acquire)); + try std.testing.expectEqual(@as(usize, 4), loader.permits.free); +} diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig new file mode 100644 index 00000000..a68e883b --- /dev/null +++ b/zml/io/direct_loader.zig @@ -0,0 +1,5361 @@ +//! Direct loading follows a batch from per-file planning through FIFO claims, +//! source reads and DMA completion. Runtime ownership and synchronization +//! live here. CUDA, ROCm, oneAPI, Furiosa and CPU load this way +//! (`backend.loadsDirect`). +//! +//! Sizing, fixed by `Loader.create`. Calibration picks the DMA block +//! (dma_calibration.zig) and maps the ring it measures in; each device's +//! DMA budget is `dma_calibration.depth` blocks. A source request is the +//! checkpoint's minimum read (`VFS.readHints`) rounded up to whole blocks, +//! one block at least (`backend.requestSize`). The source width is +//! `read_parallelism`, clipped to what the mapped ceiling fits. The +//! workspace then maps, once and before the first read, what the ring +//! leaves of `width + 1` requests, the DMA reserve and the zero block, and +//! cuts them into blocks (`Workspace.carve`): nothing maps during a load. +//! `width + 1` workers share `width` read permits, so a full width of reads +//! is in flight while one worker queues what it read. The lifecycle credits +//! are the whole pinned set in requests, so transfers that still hold +//! blocks keep the DMA stage fed and never want a block the reads did not +//! leave free. +//! +//! Planning and claims. `submit` sorts a submission's parts by file and +//! offset, plans one file at a time and publishes each plan at once, so the +//! workers read the first file while the next is planned. A plan merges +//! touching or overlapping ranges into reads, never across a gap or a file, +//! and cuts them into jobs of at most a request; jobs keep that file order, +//! and the `Scheduler` hands them out in submission order. A worker takes a +//! credit, claims a job, reads it into its slot's pinned blocks and queues +//! its preplanned transfer pieces on per-device pumps. A block may feed +//! several outputs or devices; the request keeps its credit until every +//! piece it queued completed or was abandoned. The last-transfer flag goes +//! on the piece that completes a destination's bytes, so reads and DMA +//! complete in any order. A batch owns its plans and callback contexts +//! until every job completes; a failure retires the unclaimed jobs and +//! drains what is in flight before the batch goes. +//! +//! Direct I/O. A local file the VFS reads past the page cache +//! (`VFS.useDirectIo`, asked once per file: `Planner.Reads`) is read in +//! whole alignment units: each job's read widens to aligned bounds around +//! its range, into blocks that are whole units, and its transfers address +//! the widened read. A job's range is at most a request less two units, so +//! its widened read still fits its credit's slot. Every other file is read +//! exact. The arenas are placed for how the checkpoint reads, predicted +//! before calibration maps the first (`Planner.Reads.predict`): over every +//! allowed node when at least four fifths of its bytes will read direct +//! (host_memory.zig). The files predicted then read as predicted, and every +//! other file as its own decision says. +//! +//! Destinations. A part's source bytes map to its destination's logical +//! offsets, which the destination's `ShardGrid` places on the devices by +//! arithmetic. Pieces of one destination merge across parts when block, +//! writers and both offsets are contiguous, and its parts may come from +//! different files and plans. Bytes no part covers go out with the plan of +//! its first part, as a last fill job that reads nothing and transfers from +//! the zero block, which no credit owns. The job of a tensor sharded along +//! any axis but its first is staged by destination where that saves +//! submissions (`JobCut.placeDma`). Strided parts are gathered +//! (`JobCut.appendGatherJobs`), their runs copied out of the worker's bounce +//! buffer by `Lattice.copyRuns`. Block-scale swizzles gather this way, as +//! runs of four bytes. Runs under 4 KiB cost host copies, where a program +//! costs a compile and staging memory: on DeepSeek-V4.1 over two GB300, +//! laying the MXFP4 experts out here (gate and up rows interleaved, scales +//! swizzled) copied 77 to 189 GiB through bounce buffers, 17 to 36 s of +//! worker time; as programs it copied 1 to 3 GiB and loaded 0.1 to 1.1 s +//! faster, for 4 to 6 GiB more device memory at the peak. +//! +//! Host memory. A destination in a host memory space (`Memory.isHost`) +//! takes no DMA: the worker that reads its bytes writes them in place into +//! its PJRT host buffers, which an allocation task creates ahead of the +//! reads (`Loader.startAllocations`). A host job reads straight into its +//! destinations when its read is exact and addresses each byte once, in +//! order; a widened read, or parts that overlap, goes through its slot's +//! blocks and is copied from there. Bytes no part covers are zeroed in +//! place. The devices holding one box share one allocation, through views +//! of it (`SharedHostAllocation`), so a replicated destination is written +//! once. A `host_mapped` destination's shards are always such views, in +//! the devices' default memory space, so kernels read the pinned memory in +//! place. Host and DMA parts never share a job, and no strided part goes +//! to host memory. +//! +//! Locks. Where several pumps are held, they are taken in device order. A +//! pump's mutex may be held while a released request gives its credit back +//! (`Credits`), never the reverse, and no other lock is taken under the +//! scheduler's mutex. A DMA callback, on a plugin thread, holds one pump at +//! a time. + +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zio = @import("zio"); + +const Buffer = @import("../buffer.zig").Buffer; +const pjrtx = @import("../pjrtx.zig"); +const Platform = @import("../platform.zig").Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const backend = @import("backend.zig"); +const BackendOptions = backend.Options; +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const ShardGrid = @import("ShardGrid.zig"); +const dma_calibration = @import("dma_calibration.zig"); +const Memory = @import("TensorStore.zig").Memory; +const host_memory = @import("host_memory.zig"); +const load_log = @import("log.zig").load; +const testing = @import("../testing.zig"); + +/// Creating a tensor's device state: host memory or PJRT, since the front +/// end checked its placement and memory kind. `LazyOnce` replays it to every +/// later reader of the tensor. +const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; + +/// Bounds per-device event overhead for tiny tensors. This is not a measured +/// optimum: 64 pieces smaller than block_size / 8 cannot fill an eight-block +/// byte budget. +const max_dma_pieces_per_device: usize = 64; + +/// The direct DMA backend. Submissions and awaits come from one task at a +/// time; the workers and the pumps run concurrently with them. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// The pinned arenas, grown once at creation and cut into `blocks`. + workspace: host_memory.Workspace, + /// The workspace's blocks (`carve`): the zero block first, then the + /// credits' slots (`Credits`). The zero block is the source of every DMA + /// fill transfer, in no credit's slot, never written again. + blocks: []const host_memory.Block, + scheduler: Scheduler, + metrics: Metrics = .{}, + /// One permit per source read in flight (`readSlot`), `read_width` of + /// them. The width never changes: a source that answers 429 (or 503 on + /// the object stores) is held one layer down, where the VFS holds every + /// request of that backend (`vfs/request.zig`), so the traffic falls + /// while the memory bounds stay. + read_permits: std.Io.Semaphore, + read_width: usize, + /// Lifecycle credits (`Credits`), beyond the workers' through the DMA + /// reserve. One credit beyond the width fed the DMA engines one request + /// at a time: on a GB300 loading DeepSeek-V4 (pieces of 4 MiB and 256 + /// KiB) that was 24.3 GiB/s at width 16 against 43.8 GiB/s with eight + /// requests queued, and the workers spent 9 ms per request waiting for a + /// credit against 3 ms reading. + credits: Credits, + /// The sticky first error (`recordError`). + first_error: stdx.FirstError(anyerror) = .{}, + /// One per device (`Dma.DevicePump`). Where several are locked at once + /// (`Dma.enqueue`, `Dma.retireBatch`) they are taken in device + /// order; a pump, a completion and `Dma.abortReady` hold one at a time. + pumps: []Dma.DevicePump, + /// Per-device in-flight budget: `dma_calibration.depth` blocks, in bytes. + dma_budget_bytes: usize, + worker_group: std.Io.Group = .init, + /// One per source path. Its file opens at first use (during planning + /// when a VFS decides direct IO for a local file, otherwise at the first + /// read) and stays open until `destroy`. Keyed by the slot's own copy of + /// the path: the registry that named it may go once the batch retired. + source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, + /// Device bytes of the outputs of the sealed batches, per + /// `platform.devices` index, cumulative, as `DestinationState.init` + /// allocates them; counted by the submitting task. + sealed_bytes: backend.DeviceBytes = @splat(0), + /// Device bytes allocated for outputs so far, per `platform.devices` + /// index, cumulative. Freeing an output does not decrease them. + allocated_bytes: [Platform.MAX_NUM_DEVICES]std.atomic.Value(u64) = @splat(.init(0)), + /// What the debug lines time from (`logBatch`, `logSummary`). + created_at: std.Io.Timestamp, + /// Submissions so far; the next batch's sequence number. + batch_count: usize = 0, + /// The block size, the request size and the gather limit, for the whole + /// load. + plan_config: Planner.Config, + /// How each file is read: exact, or aligned for direct I/O. + reads: Planner.Reads, + + pub fn create( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: BackendOptions, + ) !*Loader { + // A destination's shards and a transfer's writers (a u64 mask) are + // per device; Platform refuses devices past its maximum. + comptime std.debug.assert(Platform.MAX_NUM_DEVICES <= ShardGrid.max_boxes); + const self = try allocator.create(Loader); + errdefer allocator.destroy(self); + + var reads: Planner.Reads = .init(io, opts.direct_io); + errdefer reads.deinit(allocator); + // The arenas are placed for how the checkpoint reads before + // calibration maps the first: registered pages never move. + const probe_started: std.Io.Timestamp = .now(io, .awake); + const predicted = switch (platform.io_impl) { + .threaded => try reads.predict(allocator, opts.checkpoint), + // zio hands a file operation its event loop does not run (a + // real path; any, where it falls back to epoll) to a pool + // worker, and the compiles hold theirs for their whole run + // (`Compiler.compile`): the prediction takes a worker of its own + // rather than queue behind them, so the set-up overlaps them. + .zio => try zio.blockInPlaceReserved(Planner.Reads.predict, .{ &reads, allocator, opts.checkpoint }), + }; + if (opts.checkpoint != null) { + const probe_ns: u64 = @intCast(@max(probe_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("checkpoint reads predicted: files={d}, direct={Bi:.2} of {Bi:.2}, direct_reads={}, probe_ms={d:.3}", .{ + predicted.files, + predicted.direct_bytes, + predicted.bytes, + predicted.direct(), + @as(f64, @floatFromInt(probe_ns)) / std.time.ns_per_ms, + }); + } + var workspace: host_memory.Workspace = try .init(allocator, io, platform, predicted.direct()); + errdefer workspace.deinit(); + // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up + // no `start` waited for) or out of memory (the workspace logs what + // it fails to map or pin). + const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { + load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); + return err; + }; + const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, reads.alignment).?; + const blocks_per_request = request_size / block_size; + // The DMA stage of every device, kept mapped beside the source + // working set; at least one maximal source request, so a single + // request always fits beside it. + const dma_reserve = @max( + dma_calibration.depth * platform.devices.len, + Planner.blockCount(backend.max_read_request_size, block_size), + ); + // One more block for the zero fill, which no credit owns. + const zero_blocks = 1; + const available = workspace.capacity(block_size); + const width = @min( + opts.read_parallelism, + (available -| dma_reserve -| zero_blocks) / blocks_per_request -| 1, + ); + if (width == 0) { + load_log.err("host memory ceiling {Bi:.2} fits no source request beside the DMA reserve: available_blocks={d}, reserve_blocks={d}, blocks_per_request={d}: OutOfMemory", .{ host_memory.Workspace.mapped_bytes_ceiling, available, dma_reserve, blocks_per_request }); + return error.OutOfMemory; + } + if (width < opts.read_parallelism) { + load_log.debug("DMA source working set clipped by the mapped ceiling: width={d} of {d}", .{ + width, + opts.read_parallelism, + }); + } + // Two arenas, the reserve first: ROCm balances bytes across its host + // nodes per allocation, and that split is recorded. + const pregrowth_started: std.Io.Timestamp = .now(io, .awake); + const retained_before = workspace.mappedBytes(); + // The workspace logs what it fails to map. + const blocks = try workspace.carve(block_size, dma_reserve, (width + 1) * blocks_per_request + dma_reserve + zero_blocks); + const pregrowth_ns: u64 = @intCast(@max(pregrowth_started.untilNow(io, .awake).nanoseconds, 0)); + load_log.debug("host workspace pregrown: retained={Bi:.2}, pregrown={Bi:.2}, pregrowth_ms={d:.3}", .{ + workspace.mappedBytes(), + workspace.mappedBytes() - retained_before, + @as(f64, @floatFromInt(pregrowth_ns)) / std.time.ns_per_ms, + }); + std.debug.assert(blocks.len != 0); + @memset(blocks[0], 0); + var credits: Credits = try .init(allocator, blocks[1..], blocks_per_request); + errdefer credits.deinit(allocator); + const workers = width + 1; + std.debug.assert(credits.count() >= workers); + const pumps = try allocator.alloc(Dma.DevicePump, platform.devices.len); + errdefer allocator.free(pumps); + @memset(pumps, .{}); + + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .workspace = workspace, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = width }, + .read_width = width, + .credits = credits, + .pumps = pumps, + .dma_budget_bytes = dma_calibration.depth * block_size, + .created_at = .now(io, .awake), + .plan_config = .{ + .block_size = block_size, + .request_size = request_size, + .gather_limit = backend.requestSize(opts.read_hints.read_chunk_size, std.mem.min(usize, opts.dmaBlockSizes()), reads.alignment).?, + }, + .reads = reads, + }; + std.debug.assert(self.dma_budget_bytes > 0); + errdefer self.scheduler.deinit(); + + errdefer self.stopWorkers(); + for (0..workers) |_| self.worker_group.concurrent(io, workerMain, .{self}) catch |err| { + load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); + return err; + }; + load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, direct_io={t}, source_alignment={d}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ + @tagName(platform.target), + request_size, + opts.direct_io, + reads.alignment, + block_size, + self.dma_budget_bytes, + width, + self.credits.count(), + workers, + self.workspace.mappedBytes(), + }); + return self; + } + + /// Plans `specs` one source file at a time, publishes each file's plan + /// as soon as it exists behind every earlier submission, then seals the + /// batch: work on the first file starts while the rest is planned. The + /// batch owns its destinations and parts until `awaitBatch` retires it. + /// The specs keep the `LoadSpec` contract and `stridedRead`'s limit, + /// which the front end checked. A planning failure fails the loader (a + /// partial submission can never complete): the batch is awaited here + /// and the caller sees only the first error. A failed loader refuses + /// every submit with its first error. The batch's host-resident + /// destinations are then created ahead of their reads + /// (`startAllocations`). + pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { + try self.checkOpen(); + const batch = batch: { + const batch = try Batch.create(self.allocator, self.io, self.batch_count); + errdefer batch.destroy(); + try self.createDestinations(batch, specs, progress); + break :batch batch; + }; + Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config, self.reads) catch |err| { + // Fail the loader, seal and await what was published of this + // batch, and return the loader's first error. + self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); + batch.sealed_at = .now(self.io, .awake); + self.batch_count += 1; + batch.releaseUnits(1); + try self.awaitBatch(batch); + unreachable; + }; + // Stamped before the sentinel goes, so before the batch can complete. + batch.sealed_at = .now(self.io, .awake); + self.batch_count += 1; + self.countSealed(batch); + self.startAllocations(batch); + // Every plan is visible: drop the publish sentinel. A batch without + // jobs completes right here. + batch.releaseUnits(1); + return batch; + } + + /// The source bytes every job of `part` reads whole, against one read + /// (`Planner.stridedRead`). + pub fn stridedRead(self: *const Loader, part: LoadPart) backend.Backend.StridedRead { + return Planner.stridedRead(self.plan_config, self.reads, part); + } + + /// Creates the batch's host-resident destinations ahead of their reads, + /// in claim order, on a task of its own. A first-time pinned allocation + /// runs at about 14-15 GiB/s on four B70 (oneAPI), serialized whatever + /// the number of callers (one to sixteen threads measured alike; the + /// time is the xe driver's VM_BIND), so on the workers' read path it + /// stalled them for 10-12 ms per tensor; ahead of them it overlaps the + /// reads. A worker that still finds its destination missing creates it + /// itself (`LazyOnce`). The task holds one completion unit, so the batch + /// outlives it; without a task the workers create everything. + fn startAllocations(self: *Loader, batch: *Batch) void { + var host_destinations: usize = 0; + for (batch.destinations) |*destination| host_destinations += @intFromBool(destination.memory.isHost()); + if (host_destinations == 0) return; + _ = batch.remaining.fetchAdd(1, .acq_rel); + self.worker_group.concurrent(self.io, allocateMain, .{ self, batch, host_destinations }) catch |err| { + load_log.debug("batch {d}: no allocation task, workers create its {d} host destinations: {s}", .{ batch.sequence, host_destinations, @errorName(err) }); + batch.releaseUnits(1); + }; + } + + /// Walks the host jobs in claim order (the plans in publish order, each + /// plan's jobs in order) and their transfers as `run` does, skipping a + /// destination already created. `ensured` also counts one a worker was + /// still creating, which the task waited for. + fn allocateMain(self: *Loader, batch: *Batch, host_destinations: usize) void { + var ensured: usize = 0; + var previous: ?*Destination = null; + walk: for (batch.plans.items) |plan| for (plan.jobs) |job| { + switch (job.kind) { + .host_direct, .host_bounce, .host_fill => {}, + .dma_read, .dma_land, .dma_fill => continue, + } + for (plan.transfersOf(job)) |transfer| { + const destination = transfer.destination; + if (destination == previous or destination.state.readyValue() != null) continue; + previous = destination; + if (self.first_error.get() != null) break :walk; + const started = awakeNs(self.io); + _ = destination.ensureState(self) catch |err| { + self.recordError(err, "initialize tensor {s} from {s} ahead of its reads, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + break :walk; + }; + _ = self.metrics.ahead_init_ns.fetchAdd(awakeNs(self.io) -| started, .monotonic); + ensured += 1; + } + }; + load_log.debug("batch allocations: batch={d}, host_destinations={d}, ensured={d}, done=+{d:.3}s", .{ batch.sequence, host_destinations, ensured, secondsBetween(self.created_at, .now(self.io, .awake)) }); + batch.releaseUnits(1); + } + + /// Waits for the batch's last completion unit, retires it and returns + /// the loader's sticky error if the loader failed. Destroying the batch + /// destroys its transfer managers, and PJRT's manager destructor errors + /// every buffer whose last transfer never went out, so a buffer the + /// failure left open reports that error, never success. A transfer + /// that fails asynchronously errors its buffer's definition event too + /// (the done event carries no error). A program reading such a buffer + /// fails its execution; a bulk buffer shows the error at its first use. + /// Either way an output of a failed submission holds buffers whose + /// contents are unspecified, or was never written. A batch that + /// completed without an error submitted every byte of every target (the + /// pump flagged the submission that completed its bytes); one that did + /// not would leave a buffer PJRT then errors, a bookkeeping bug this + /// panics on. A host-resident destination has no target: its workers + /// write it in place, which only the debug check of its progress covers. + pub fn awaitBatch(self: *Loader, batch: *Batch) !void { + batch.done.waitUncancelable(self.io); + const done_at: std.Io.Timestamp = .now(self.io, .awake); + // Every request of this batch has completed: no worker or callback + // touches its managers or contexts any more. + const load_error = self.first_error.get(); + if (load_error == null) for (batch.destinations) |*destination| { + const submitted = submitted: { + const state = destination.state.readyValue() orelse break :submitted false; + for (state.targets) |target| { + if (!target.fullySubmitted()) break :submitted false; + } + break :submitted true; + }; + if (!submitted) std.debug.panic("loader: batch {d} completed without submitting every byte of tensor {s} {f}", .{ batch.sequence, destination.name(), destination.shape }); + }; + // Each destination completed its progress item. + if (load_error == null) for (batch.destinations) |*destination| std.debug.assert(destination.unread_bytes.load(.acquire) == 0); + Dma.retireBatch(self, batch); + self.logBatch(batch, done_at, load_error == null); + self.destroyBatch(batch); + if (load_error) |err| return err; + } + + /// Fails the loader unless it already failed (`recordError`): unclaimed + /// jobs retire, queued pieces drain, and requests in flight finish + /// their read. + pub fn abort(self: *Loader) void { + self.recordError(error.Canceled, "loader aborted", .{}); + } + + /// The caller awaited every batch before this; nothing is queued or in + /// flight, so stopping the workers is a plain shutdown. + pub fn destroy(self: *Loader) void { + self.stopWorkers(); + self.logSummary(); + var slots = self.source_slots.valueIterator(); + while (slots.next()) |slot| { + slot.*.deinit(self.io); + self.allocator.free(slot.*.uri); + self.allocator.destroy(slot.*); + } + self.source_slots.deinit(self.allocator); + self.reads.deinit(self.allocator); + std.debug.assert(self.credits.inUse(self.io) == 0); + for (self.pumps) |*device_pump| { + std.debug.assert(device_pump.active_pieces == 0); + std.debug.assert(device_pump.queue.len == 0); + std.debug.assert(device_pump.retired == null); + device_pump.queue.deinit(self.allocator); + } + self.allocator.free(self.pumps); + self.scheduler.deinit(); + self.credits.deinit(self.allocator); + self.workspace.deinit(); + self.allocator.destroy(self); + } + + /// Output bytes of the sealed batches not allocated yet, per + /// `platform.devices` index. Workers allocate as they read, possibly + /// before the batch is sealed, so the difference saturates. + pub fn unallocatedBytes(self: *const Loader) backend.DeviceBytes { + var unallocated = self.sealed_bytes; + for (&unallocated, &self.allocated_bytes) |*bytes, *allocated| bytes.* -|= allocated.load(.acquire); + return unallocated; + } + + /// Counts the device bytes of the sealed `batch`: a box of each + /// destination outside host memory on each of its devices, as + /// `DestinationState.init` allocates them. + fn countSealed(self: *Loader, batch: *const Batch) void { + for (batch.destinations) |*destination| { + if (destination.memory.isHost()) continue; + for (destination.grid.devices) |device| self.sealed_bytes[device.id] += destination.grid.box_bytes; + } + } + + fn checkOpen(self: *Loader) !void { + if (self.first_error.get()) |err| return err; + } + + /// Creates the batch's destinations, one per spec, and their parts in + /// its arena, their sources and dimensions staying the specs' until the + /// batch is awaited (`backend.LoadSpec`). A destination's device state + /// is created lazily by the first worker that reads or fills for it, + /// or by the allocation task. + fn createDestinations( + self: *Loader, + batch: *Batch, + specs: []const LoadSpec, + progress: std.Progress.Node, + ) !void { + const arena = batch.arena.allocator(); + var part_count: usize = 0; + for (specs) |spec| part_count += spec.parts.len; + const destinations = try arena.alloc(Destination, specs.len); + const parts = try arena.alloc(Part, part_count); + var next_part: usize = 0; + for (specs, destinations) |spec, *destination| { + const grid: ShardGrid = .init(self.platform, spec.shape); + const destination_parts = parts[next_part..][0..spec.parts.len]; + next_part += spec.parts.len; + var covered_bytes: u64 = 0; + var gathered = false; + for (spec.parts, destination_parts) |part, *slot| { + slot.* = .{ + .source = part.source, + .source_slot = try self.sourceSlot(part.source.file_uri), + .destination = destination, + .lattice = part.lattice, + }; + covered_bytes += part.lattice.width; + gathered = gathered or part.lattice.dims.len != 0; + const strided = self.stridedRead(part); + std.debug.assert(strided.read <= strided.limit); // backend.Backend.stridedRead + } + std.debug.assert(!gathered or !spec.memory.isHost()); // backend.LoadSpec.memory + destination.* = .{ + .shape = spec.shape, + .grid = grid, + .memory = spec.memory, + .output = spec.output, + .parts = destination_parts, + .gathered = gathered, + // A gathered destination's jobs land every byte, zeros too. + .unread_bytes = .init(if (gathered) grid.total else covered_bytes), + .progress = progress, + }; + } + batch.destinations = destinations; + batch.parts = parts; + } + + /// Releases the destinations' device state, then the batch's arena. + fn destroyBatch(self: *Loader, batch: *Batch) void { + for (batch.destinations) |*destination| destination.deinit(self.allocator, self.platform.pjrt_api); + batch.destroy(); + } + + fn sourceSlot(self: *Loader, uri: []const u8) !*SourceSlot { + if (self.source_slots.get(uri)) |slot| return slot; + const owned = try self.allocator.dupe(u8, uri); + errdefer self.allocator.free(owned); + const slot = try self.allocator.create(SourceSlot); + errdefer self.allocator.destroy(slot); + slot.* = .{ .uri = owned }; + try self.source_slots.putNoClobber(self.allocator, owned, slot); + return slot; + } + + fn workerMain(self: *Loader) void { + var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot, self.plan_config.block_size) catch |err| { + self.recordError(err, "allocate source worker scratch: blocks={d}", .{self.credits.blocks_per_slot}); + return; + }; + defer scratch.deinit(); + while (self.scheduler.waitForWork(self.io)) { + const request = self.claimRequest() orelse continue; + // The scheduling sentinel keeps the batch, and with it the + // request's plan, alive through `run` and error reporting. + defer request.release(); + request.run(&scratch) catch |err| { + self.recordError(err, "source request: file={s}, job={t}", .{ request.plan.source_slot.uri, request.plan.jobs[request.index].kind }); + }; + } + } + + /// Takes a credit, then the next job (`Credits`): a claimed job never + /// waits for its slot. Null, with the credit given back, when nothing + /// was left to claim. + fn claimRequest(self: *Loader) ?*ReadRequest { + const credit_wait_started = awakeNs(self.io); + const slot = self.credits.acquire(self.io); + const credit_wait_ns = awakeNs(self.io) -| credit_wait_started; + const claim = self.scheduler.claim(self.io) orelse { + self.credits.release(self.io, slot); + return null; + }; + _ = self.metrics.credit_wait_ns.fetchAdd(credit_wait_ns, .monotonic); + return ReadRequest.init(self, claim, slot); + } + + /// Stops the workers: every batch was awaited, so nothing is queued and + /// no request holds a credit or permit, and `Scheduler.fail` only wakes + /// the workers waiting for work. A worker may still hold the credit it + /// took for a claim that finds nothing (`claimRequest`); it gives it + /// back before it sees `stopping`, so every credit is free once this + /// returns. Awaiting a group that never spawned returns at once, so + /// this also cleans up a failed `create`. + fn stopWorkers(self: *Loader) void { + std.debug.assert(self.scheduler.queue.len == 0); + self.scheduler.fail(self.io); + self.worker_group.await(self.io) catch {}; + } + + /// Keeps `err` as the sticky error unless one is kept already and logs + /// it, then fails the load without waiting: unclaimed jobs retire + /// (`Scheduler.fail`) and queued pieces are released + /// (`Dma.abortReady`). Any thread may call it, a DMA callback included; + /// requests in flight stop at their next read (`readSlot`) or enqueue + /// and release their units. + /// Failing other batches than the caller's is safe from any thread: + /// `Scheduler.fail` and `Dma.abortReady` reach only live batches (their + /// docs), and `destroy` waits for the worker group before it frees what + /// the workers use. + fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { + if (self.first_error.set(err)) { + load_log.failure(err, fmt, args); + self.scheduler.fail(self.io); + Dma.abortReady(self); + } + } + + /// Counts the plans' jobs, transfers and events only when the lines + /// would be logged. + fn logBatch(self: *Loader, batch: *const Batch, done_at: std.Io.Timestamp, successful: bool) void { + if (!std.log.logEnabled(.debug, .@"zml/io/load")) return; + const sealed_at = batch.sealed_at orelse self.created_at; + const published_at = batch.published_at orelse sealed_at; + var logical_bytes: usize = 0; + for (batch.destinations) |*destination| logical_bytes += destination.shape.byteSize(); + load_log.debug("batch completed: batch={d}, successful={}, logical_bytes={Bi:.2}, published=+{d:.3}s, sealed=+{d:.3}s, done=+{d:.3}s, elapsed={d:.3}s, source_width={d}, request_size={Bi:.2}", .{ + batch.sequence, + successful, + logical_bytes, + secondsBetween(self.created_at, published_at), + secondsBetween(self.created_at, sealed_at), + secondsBetween(self.created_at, done_at), + secondsBetween(published_at, done_at), + self.read_width, + self.plan_config.request_size, + }); + + var planning_ns: u64 = 0; + var source_bytes: u64 = 0; + var jobs: usize = 0; + var transfers: usize = 0; + var dma_submissions: usize = 0; + var kinds: std.EnumArray(std.meta.Tag(Plan.Job.Kind), usize) = .initFill(0); + var gather_jobs: usize = 0; + for (batch.plans.items) |plan| { + planning_ns += plan.planning_ns; + source_bytes += plan.source_bytes; + jobs += plan.jobs.len; + transfers += plan.transfers.len; + dma_submissions += plan.events.len; + for (plan.jobs) |job| { + kinds.getPtr(job.kind).* += 1; + // A gathered destination's landing jobs land it alone. + if (job.kind == .dma_land and plan.transfersOf(job)[0].destination.gathered) gather_jobs += 1; + } + } + const fill_jobs = kinds.get(.dma_fill) + kinds.get(.host_fill); + const host_jobs = kinds.get(.host_direct) + kinds.get(.host_bounce) + kinds.get(.host_fill); + const source_jobs = jobs - fill_jobs; + const average_read_size = if (source_jobs == 0) 0 else source_bytes / source_jobs; + const coalescing_ratio = if (source_jobs == 0) + 0 + else + @as(f64, @floatFromInt(batch.parts.len)) / + @as(f64, @floatFromInt(source_jobs)); + load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, host_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ + batch.sequence, + batch.plans.items.len, + @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, + source_bytes, + source_jobs, + fill_jobs, + host_jobs, + batch.parts.len, + transfers, + dma_submissions, + kinds.get(.dma_land) - gather_jobs, + gather_jobs, + coalescing_ratio, + average_read_size, + }); + } + + fn logSummary(self: *Loader) void { + const reads = self.metrics.read_operations.load(.acquire); + load_log.debug("loader summary: batches={d}, successful={}, read_bytes={Bi:.2}, elapsed={d:.3}s, reads={d}, source_width={d}, request_size={Bi:.2}, credits_high_water={d} of {d}, pinned_mapped={Bi:.2}", .{ + self.batch_count, + self.first_error.get() == null, + self.metrics.read_bytes.load(.acquire), + secondsBetween(self.created_at, .now(self.io, .awake)), + reads, + self.read_width, + self.plan_config.request_size, + self.credits.high_water, + self.credits.count(), + self.workspace.mappedBytes(), + }); + load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}, ahead_init_ms={d:.3}", .{ + millisecondsPer(self.metrics.credit_wait_ns.load(.acquire), reads), + millisecondsPer(self.metrics.read_ns.load(.acquire), reads), + millisecondsPer(self.metrics.dma_stage_ns.load(.acquire), reads), + millisecondsPer(self.metrics.tensor_init_ns.load(.acquire), reads), + millisecondsPer(self.metrics.ahead_init_ns.load(.acquire), 1), + }); + const submissions = self.metrics.dma_submissions.load(.acquire); + load_log.debug("loader DMA: dma_submissions={d}, dma_submit_us_per_piece={d:.2}, dma_piece_latency_ms={d:.3}, pump_stops_empty={d}, pump_stops_full={d}", .{ + submissions, + millisecondsPer(self.metrics.dma_submit_ns.load(.acquire), submissions) * 1000, + millisecondsPer(self.metrics.dma_piece_ns.load(.acquire), submissions), + self.metrics.pump_stops_empty.load(.acquire), + self.metrics.pump_stops_full.load(.acquire), + }); + } +}; + +/// One submission: its plans (one per source file, published as each file +/// is planned), the destinations it writes and their parts, and every +/// request and event context created while loading it, all in the batch's +/// arena until it retires. `remaining` counts completion units: one per +/// published job plus a publish sentinel held until the submission is +/// sealed, and one for the allocation task while it runs +/// (`Loader.startAllocations`), taken before the sentinel is dropped. A +/// job's unit is released exactly once, by whichever of these happens: its +/// request's last reference drops (final DMA callback, a host job's end, or +/// abandonment), or `Scheduler.fail` retires it unclaimed. The batch is +/// done when `remaining` reaches zero; the awaiting task then retires it, +/// so releasing a unit is the last permitted access to the batch. +pub const Batch = struct { + /// Holds the plans with their contexts, the destinations and the parts + /// until `destroy`; only the submitting task allocates in it. + arena: std.heap.ArenaAllocator, + io: std.Io, + /// Published plans in file order: appended by the submitting task under + /// the scheduler mutex. + plans: std.ArrayList(*Plan) = .empty, + /// Their device state is the loader's, released before `destroy`. + destinations: []Destination = &.{}, + /// The destinations' parts, each destination's consecutive and in its + /// destination order: what the planner sorts by file offset. + parts: []Part = &.{}, + remaining: std.atomic.Value(usize), + done: std.Io.Event = .unset, + /// Submission number within the loader, for log correlation. + sequence: usize, + /// The first publish; null for a submission without plans. + published_at: ?std.Io.Timestamp = null, + /// Once every plan is published, before the publish sentinel goes. + sealed_at: ?std.Io.Timestamp = null, + + /// An open batch holding its publish sentinel and nothing else. The + /// sentinel keeps the batch from completing before its last plan is + /// visible. + fn create(allocator: std.mem.Allocator, io: std.Io, sequence: usize) !*Batch { + const self = try allocator.create(Batch); + self.* = .{ + .arena = .init(allocator), + .io = io, + .remaining = .init(1), + .sequence = sequence, + }; + return self; + } + + /// Releases `count` completion units. MEMORY-ORDER RULE: this must be the + /// caller's final access to the batch and to anything it owns (requests, + /// events, destinations, parts, plans; a request's slot of blocks is the + /// loader's and went back with its credit just before). The last unit + /// sets `done`, and the awaiting task frees all of it as soon as it + /// observes the event. + fn releaseUnits(self: *Batch, count: usize) void { + if (count == 0) return; + const previous = self.remaining.fetchSub(count, .acq_rel); + std.debug.assert(previous >= count); + if (previous == count) self.done.set(self.io); + } + + /// Frees the arena: the plans with their contexts, which + /// `Dma.retireBatch` must already have retired, the destinations and + /// the parts. + fn destroy(self: *Batch) void { + const allocator = self.arena.child_allocator; + self.arena.deinit(); + allocator.destroy(self); + } +}; + +/// One output buffer and what places it. Owned by the batch. Its parts may +/// come from several files, so several plans can write it: their pieces, +/// and the zero fill, all count toward the same targets (`Target.total` is +/// the placement's bytes, whichever plan submits them), and its device +/// state is created once, by the first worker that reads or fills for it +/// or, for a host-resident one, normally by the allocation task ahead of +/// them. +const Destination = struct { + /// The caller's shape, which the output buffer keeps. + shape: Shape, + /// Where `shape`, packed and placed (`Platform.placedShape`), lands on + /// its devices: what PJRT allocates and the planner cuts. + grid: ShardGrid, + /// Where its bytes go (`TensorStore.Memory`). + memory: Memory = .default, + output: *Buffer, + /// What fills it, in ascending, disjoint destination order + /// (`backend.LoadSpec`): a slice of the batch's parts. Bytes no part + /// covers are written from the loader's zero block, by the fill job of + /// the plan holding `parts[0]`. + parts: []Part, + /// Fed by strided parts: planned whole, by logical ranges, with the + /// plan holding `parts[0]` (`Planner.JobCut.appendGatherJobs`), whose + /// jobs also write its zero fill. + gathered: bool = false, + /// Bytes not read yet: the parts' widths, or for a gathered destination + /// its whole placed size (its gather jobs land the zeros too). The read + /// that brings it to zero completes the destination's item on + /// `progress`. + unread_bytes: std.atomic.Value(u64), + progress: std.Progress.Node = .none, + state: LazyOnce(DestinationState) = .{}, + + /// How its parts are planned, a class at a time in this order + /// (`Planner.sortedPartOrder`): DMA jobs, host jobs, or gathered whole. + const Delivery = enum { dma, host, gathered }; + + fn delivery(self: *const Destination) Delivery { + if (self.gathered) return .gathered; + return if (self.memory.isHost()) .host else .dma; + } + + fn ensureState(self: *Destination, direct: *Loader) StateInitError!*DestinationState { + return self.state.ensure(direct.io, DestinationState.init, .{ direct, self }); + } + + fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + if (self.state.readyValue()) |state| state.deinit(allocator, api); + } + + /// The first source's name, for logs. + fn name(self: *const Destination) []const u8 { + return self.parts[0].source.name; + } + + /// Counts `bytes` as read (`unread_bytes`); the last read completes the + /// destination's one progress item. + fn recordRead(self: *Destination, bytes: u64) void { + if (self.unread_bytes.fetchSub(bytes, .acq_rel) == bytes) self.progress.completeOne(); + } +}; + +/// One source range of a destination: what the planner sorts by file offset +/// and cuts into jobs. A plain tensor is one part covering its whole +/// destination. +const Part = struct { + source: *const safetensors.Tensor, + source_slot: *SourceSlot, + destination: *Destination, + /// The spec part's; its dimensions are borrowed until the await + /// (`backend.LoadSpec`). + lattice: backend.Lattice, + + /// The part's bytes in its file, which `backend.checkParts` bounds. + fn fileRange(self: *const Part) struct { start: u64, end: u64 } { + const start = self.source.offset + self.lattice.src_offset; + return .{ .start = start, .end = start + self.lattice.width }; + } +}; + +/// One source file: every job of a plan reads the file of its slot, which +/// the first read opens, or the planner when it asks the VFS whether the +/// file is read directly (`Planner.Reads`). +const SourceSlot = struct { + uri: []const u8, + file: LazyOnce(std.Io.File) = .{}, + /// The file's read alignment once the planner decided it + /// (`Planner.Reads`): 0 when it is read exactly. + alignment: ?usize = null, + + fn ensure(self: *SourceSlot, io: std.Io) std.Io.File.OpenError!std.Io.File { + const file = try self.file.ensure(io, std.Io.Dir.openFile, .{ .cwd(), io, self.uri, .{ .mode = .read_only } }); + return file.*; + } + + fn deinit(self: *SourceSlot, io: std.Io) void { + if (self.file.readyValue()) |file| file.close(io); + } +}; + +/// One file's jobs in claim order with their transfer records and every +/// context the jobs can need, allocated by the planner: one request per +/// job and one event per planned DMA submission, handed out in +/// submission order; a host job needs only its request. Contexts hold the +/// plan's address, which the batch's arena keeps until it retires. +const Plan = struct { + /// A claimable job: its slice of the plan's transfers and what its + /// worker does for them. Its request slot is its index. + const Job = struct { + /// Where the job reads first: its read's start, a landing job's + /// first extent's; zero for a job that reads nothing. + /// `Planner.preparePlan` sorts the other jobs by it and appends + /// the fill jobs after them. + file_offset: u64, + transfer_start: usize, + transfer_len: usize, + kind: Kind, + + /// How the job's bytes reach their destinations. A destination + /// in a host memory space (`Memory.isHost`) is written + /// by the worker that reads it: going through pinned blocks and + /// a host-to-host PJRT transfer copied every byte twice and held + /// the DMA pumps, which on four B70 made Llama-3.1-8B take 2.0 s + /// into pinned_host against 0.67 s into device memory. + const Kind = union(enum) { + /// Read into pinned blocks in file order, where its transfers + /// address it, then copied to the devices by DMA. + dma_read: Read, + /// Reads its extents, whose items land their bytes in its + /// blocks: a read of its plan's file grouped by destination + /// box (`Planner.JobCut.stage`), or a gathered destination's + /// range (`Planner.JobCut.appendGatherJobs`). + dma_land: Land, + /// Reads nothing: every transfer goes from the loader's zero + /// block (`Planner.JobCut.appendFillTransfers`). + dma_fill, + /// Read straight into host-resident destinations: the read's + /// iovecs are the destination ranges, so the read is the only + /// copy. Needs a read without alignment padding whose + /// transfers address each of its bytes once, in order + /// (`Planner.JobCut.coversReadInOrder`). + host_direct: Read, + /// Read into the request's slot of blocks, then copied into + /// host-resident destinations by the worker: reads widened + /// for direct I/O, whose buffers must be aligned, and + /// overlapping parts. + host_bounce: Read, + /// Writes zeros in place, reading nothing. + host_fill, + }; + + /// `len` bytes from the job's `file_offset`, of which the first + /// `minimum_len` hold tensor data and must exist. The rest is + /// alignment padding the end of the file may cut. + const Read = struct { + len: usize, + minimum_len: usize, + }; + + /// A landing job: `len` bytes in its blocks, landed by the + /// `extent_len` extents of the plan's `extents` from + /// `extent_start`, in order. A gathered range no part covers + /// has none. + const Land = struct { + len: usize, + extent_start: usize, + extent_len: usize, + /// Its blocks start zeroed: bytes no part covers. + zeroed: bool = false, + }; + + /// The blocks of its credit's slot the job reads into or lands + /// in; none for a direct host job or a fill job. + fn blockLen(self: Job, block_size: usize) usize { + return switch (self.kind) { + .dma_read, .host_bounce => |read| Planner.blockCount(read.len, block_size), + .dma_land => |land| Planner.blockCount(land.len, block_size), + .dma_fill, .host_direct, .host_fill => 0, + }; + } + }; + + /// Bytes an extent of a landing job places in its blocks: the runs + /// `first` to `end` of `lattice`, its source offsets in the file + /// and its destination offsets logical, clipped to the logical + /// range `[clip_start, clip_end)`. A byte at logical offset `l`, + /// which `grid.locate(l)` puts at offset `o` of box `b` of the + /// destination's grid, lands at `landings[landing_start + b] +% o` + /// of the job's blocks taken end to end (`walkItem`). + const Item = struct { + destination: *Destination, + lattice: backend.Lattice, + first: u64 = 0, + end: u64 = 1, + clip_start: u64, + clip_end: u64, + landing_start: usize, + }; + + /// One read of a landing job: `len` bytes from `file_offset` of + /// `source_slot`'s file, of which the first `minimum_len` exist, + /// landed by the `item_len` items of the plan's `items` from + /// `item_start`. + const Extent = struct { + source_slot: *SourceSlot, + file_offset: u64, + len: usize, + minimum_len: usize, + /// Read straight into the job's blocks: an exact read whose + /// items' runs tile it, in order. Otherwise read into the + /// worker's bounce buffer and copied from there. + scatter: bool, + item_start: usize, + item_len: usize, + }; + + /// `len` bytes at `landing` of the job's blocks taken end to end, + /// for every device of `writer_mask` at `destination_offset` of its + /// shard. Every transfer stays inside one block, which the worker or + /// `Dma.enqueue` finds from `landing` (`blockPiece`); a direct host + /// job reads at the same landings without blocks. A fill one lands + /// at 0: a DMA one reads the zero block, a host one writes zeros. + const Transfer = struct { + destination: *Destination, + landing: usize, + writer_mask: u64, + destination_offset: usize, + len: usize, + }; + + /// The submission this plan belongs to; set when it is published, + /// under the scheduler mutex, with its completion units. + batch: *Batch = undefined, + /// The file every job of the plan reads. + source_slot: *SourceSlot, + jobs: []Job, + transfers: []Transfer, + /// The landing jobs' extents, their items and the items' landing + /// tables (one entry per box of the item's destination). + extents: []Extent = &.{}, + items: []Item = &.{}, + landings: []u64 = &.{}, + requests: []ReadRequest, + events: []Dma.EventContext, + /// Event slots handed out so far, one `fetchAdd` per submission from + /// any device's pump. + events_used: std.atomic.Value(usize) = .init(0), + /// The tensor bytes the jobs read. + source_bytes: u64, + /// How long the plan took to cut. + planning_ns: u64 = 0, + /// Next job to claim; owned by the scheduler mutex. + cursor: usize = 0, + + fn transfersOf(self: *const Plan, job: Job) []const Transfer { + return self.transfers[job.transfer_start..][0..job.transfer_len]; + } +}; + +/// Builds immutable per-file source jobs and transfer records. Their claim +/// order (file order) is decided here once. +/// For DeepSeek-V4-Flash (148.65 GiB, 69,187 tensors), tensor-local reads made +/// about 69,445 source calls despite a 16 MiB request limit. Coalescing reduced +/// that to 9,524. A rigid grid still produced 78,665 DMA pieces; tensor-safe +/// cuts kept the same read count with 69,572 pieces. Per-tensor device buffers +/// remain intentional: reducing submissions below roughly tensors x destinations would require +/// packed device allocations or device-side scattering, not another read cut. +const Planner = struct { + /// What every plan of a loader is cut by. + const Config = struct { + block_size: usize, + /// The longest read, a whole number of blocks + /// (`backend.requestSize`). + request_size: usize, + /// The one-read limit of a strided part (`stridedRead`, which caps + /// it at `request_size`): the request of the smallest block + /// calibration may pick, so a refusal does not follow its pick. + gather_limit: usize = std.math.maxInt(usize), + /// Submissions a DMA job must save to be staged (`JobCut.stage`): + /// one device's in-flight pieces. + minimum_staging_gain: usize = max_dma_pieces_per_device, + }; + + /// How each file is read: aligned when it is local and the VFS that + /// opens it reads it directly under `policy`, or as `predict` answered + /// for it, decided once per file and kept in its slot. A remote file is + /// read exact without being opened here. Without a VFS (`io` is none, or + /// no alignment) every file takes `alignment`. A gather job reads files + /// other than its plan's, so it asks here for each. `predict` tells, + /// before any file is planned, how the checkpoint's files will read. + /// Widening buffered reads is not free: a widened read served from the + /// page cache costs the DMA sources their block alignment for nothing, + /// and warm replicated Llama on four GB300 lost 3-6% when its first + /// tensor no longer landed at block offset zero. So a file asks the VFS + /// actually owning its handle (`VFS.fromIo`, in `init`), both to avoid + /// that cost and to avoid indexing another VFS's handle table; an `io` + /// that is no VFS never reads directly. + const Reads = struct { + io: std.Io, + vfs: ?*VFS, + policy: VFS.DirectIo, + /// Every read of a file read directly starts and ends at a multiple + /// of this; 0 reads every file exact. + alignment: usize, + /// The files `predict` answered for, by real path as the registry + /// names them (`safetensors.Tensor.file_uri`), each with the policy + /// that decides it as answered: under `auto`, `on` for a file that + /// reads direct and `off` for one that reads buffered. Under `on`, + /// which samples nothing, every file keeps `on`, and a filesystem + /// refusing the flag is logged when the file is planned. Owned until + /// `deinit`. + predicted: std.StringHashMapUnmanaged(VFS.DirectIo) = .empty, + + fn init(io: std.Io, policy: VFS.DirectIo) Reads { + const alignment = backend.readAlignment(io, policy); + return .{ .io = io, .vfs = if (alignment != 0) VFS.fromIo(io) else null, .policy = policy, .alignment = alignment }; + } + + fn deinit(self: *Reads, allocator: std.mem.Allocator) void { + var paths = self.predicted.keyIterator(); + while (paths.next()) |path| allocator.free(path.*); + self.predicted.deinit(allocator); + } + + /// The file's alignment, decided the first time it is asked: a + /// local file is opened here, and decided as `predict` answered for + /// it, else under `policy`. One that fails to open is read exact: + /// its reads fail with the same error (`LazyOnce`), which `readSlot` + /// logs with the path. + fn alignmentOf(self: Reads, slot: *SourceSlot) error{Canceled}!usize { + const vfs = self.vfs orelse return self.alignment; + if (slot.alignment) |known| return known; + // A predicted file is decided as predicted, not sampled again: + // the prediction's sample queued the read of each page it missed + // and of the kernel's readahead, and a second sample of those + // pages once the reads have landed finds the file cached + // (`VFS.predictDirectIo`). Under `on`, a filesystem refusing the + // flag still reads it buffered. + const policy = self.predicted.get(slot.uri) orelse self.policy; + const direct = vfs.isLocal(slot.uri) and vfs.useDirectIo(slot.ensure(self.io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return 0, + }, policy); + const alignment = if (direct) self.alignment else 0; + slot.alignment = alignment; + return alignment; + } + + /// The widest alignment `alignmentOf` can give the file, without + /// opening it: 0 for a remote file. + fn boundOf(self: Reads, uri: []const u8) usize { + const vfs = self.vfs orelse return self.alignment; + return if (vfs.isLocal(uri)) self.alignment else 0; + } + + /// How the checkpoint's local safetensors files will read, asked + /// once, before any is planned, to place the arenas: each as + /// `VFS.predictDirectIo` answers under `policy`, an answer kept in + /// `predicted` for `alignmentOf` to decide the file by. The files + /// are those `TensorRegistry.fromPath` reads for the checkpoint + /// (`safetensors.checkpointFiles`, which reads its index). Nothing + /// without a VFS, for a checkpoint that is not local, or for files + /// that do not resolve or open: the registry reports those. Keeps + /// no file open, and logs nothing. A wrong prediction (residency + /// changing before the file is planned) costs speed, never bytes. + fn predict(self: *Reads, allocator: std.mem.Allocator, checkpoint: ?[]const u8) error{ Canceled, OutOfMemory }!Prediction { + std.debug.assert(self.predicted.count() == 0); + const vfs = self.vfs orelse return .{}; + const path = checkpoint orelse return .{}; + if (!vfs.isLocal(path)) return .{}; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const dir, const names = safetensors.checkpointFiles(arena.allocator(), self.io, path) catch |err| switch (err) { + error.Canceled => return error.Canceled, + error.OutOfMemory => return error.OutOfMemory, + else => return .{}, + }; + defer dir.close(self.io); + var prediction: Prediction = .{}; + var real_path_buffer: [std.fs.max_path_bytes]u8 = undefined; + for (names) |name| { + const real_path_len = dir.realPathFile(self.io, name, &real_path_buffer) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => continue, + }; + const real_path = real_path_buffer[0..real_path_len]; + // A symlink and the file it leads to are sampled once. + if (self.predicted.contains(real_path)) continue; + const file = try vfs.predictDirectIo(dir, name, self.policy) orelse continue; + const owned = try allocator.dupe(u8, real_path); + errdefer allocator.free(owned); + try self.predicted.putNoClobber(allocator, owned, if (file.direct or self.policy == .on) .on else .off); + prediction.add(file); + } + return prediction; + } + + /// The bytes of a checkpoint's local safetensors files, and of those + /// that will read direct (`predict`). + const Prediction = struct { + files: usize = 0, + bytes: u64 = 0, + direct_bytes: u64 = 0, + + fn add(self: *Prediction, file: VFS.DirectIoPrediction) void { + self.files += 1; + self.bytes += file.size; + if (file.direct) self.direct_bytes += file.size; + } + + /// Whether the arenas are placed for direct reads + /// (`host_memory.Workspace.init`): files holding at least four + /// fifths of the bytes read direct. In measurements on two GB300 + /// of one Grace node loading Qwen3.8-27B-FP8 (28.75 GiB), + /// spreading the arenas over both nodes saved direct reads about + /// 4.2 ms per GiB forced direct and 4.0 cold under auto, and cost + /// reads through the page cache 10 to 16 ms per GiB: it pays from + /// about 0.71 to 0.80 of the bytes read direct, and four fifths, + /// at the top of that range, leans to the devices' nodes, whose + /// mistake costs less per GiB. + fn direct(self: Prediction) bool { + return self.bytes != 0 and self.direct_bytes * 5 >= self.bytes * 4; + } + }; + }; + + /// Plans `parts` one file at a time and publishes each plan as soon as + /// it exists, so workers claim the first file while the rest is planned, + /// each plan in the batch's arena. Stops at the first error; the caller + /// seals or fails the batch. `reads` says how each file is read + /// (`Reads`). + /// Whole-model planning delayed the first read by 0.32 s for DeepSeek's + /// 46 files on one MI300X; publishing per file overlaps the rest with IO. + fn publishFiles( + scheduler: *Scheduler, + io: std.Io, + batch: *Batch, + parts: []Part, + config: Config, + reads: Reads, + ) !void { + const order = try sortedPartOrder(scheduler.allocator, parts); + defer scheduler.allocator.free(order); + var file_start: usize = 0; + while (file_start < order.len) { + const file_end = fileGroupEnd(parts, order, file_start); + const planning_started: std.Io.Timestamp = .now(io, .awake); + const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config, reads); + plan.planning_ns = @intCast(@max(planning_started.untilNow(io, .awake).nanoseconds, 0)); + try scheduler.publish(io, batch, plan); + if (batch.published_at == null) batch.published_at = .now(io, .awake); + file_start = file_end; + } + } + + /// Plans one file: `order` indexes its `parts` by delivery class, then + /// offset and size (`sortedPartOrder`). `JobCut.cutRuns` cuts the DMA + /// and the host parts at the file's alignment (`Reads.alignmentOf`), + /// `JobCut.appendGatherJobs` each gathered destination with the file of + /// its first part; the jobs merge in file order, then + /// `JobCut.appendFillJob` appends each class's zero fill. The plan goes + /// in `arena` (`JobCut.finish`), the cut in `allocator`. + fn preparePlan( + allocator: std.mem.Allocator, + arena: std.mem.Allocator, + parts: []Part, + order: []const usize, + config: Config, + reads: Reads, + ) !*Plan { + const source_slot = parts[order[0]].source_slot; + for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); + const dma_end = deliveryEnd(parts, order, 0, .dma); + const host_end = deliveryEnd(parts, order, dma_end, .host); + const alignment = try reads.alignmentOf(source_slot); + var cut: JobCut = .{ + .allocator = allocator, + .parts = parts, + .source_slot = source_slot, + .config = config, + .reads = reads, + .alignment = alignment, + .maximum_job_len = maximumJobLen(config.request_size, alignment), + // A gather job's extents may be in files read directly while + // this one is not, so its ranges leave room for the widest + // alignment. + .gather_len = maximumJobLen(config.request_size, reads.alignment), + }; + defer cut.deinit(); + try cut.cutRuns(order[0..dma_end], false); + try cut.cutRuns(order[dma_end..host_end], true); + for (order[host_end..]) |part_index| { + const part = &parts[part_index]; + std.debug.assert(part.destination.gathered); + if (part == &part.destination.parts[0]) try cut.appendGatherJobs(part.destination); + } + // Jobs are claimed in planning order, which is file order (a + // gather job by its first read). A byte-fair order across + // destination devices was measured against it on the two hosts + // where the DMA pump is not the ceiling and made no difference. + // Each class is in file order already; the stable sort interleaves + // them. + std.mem.sort(Plan.Job, cut.jobs.items, {}, struct { + fn lessThan(_: void, lhs: Plan.Job, rhs: Plan.Job) bool { + return lhs.file_offset < rhs.file_offset; + } + }.lessThan); + + // The zero fill of each destination goes out once, with the plan + // of its first part; it depends on no read, so one job after this + // file's reads carries all of it, one per delivery class. + try cut.appendFillJob(order[0..dma_end], false); + try cut.appendFillJob(order[dma_end..host_end], true); + return cut.finish(arena); + } + + /// The end of the parts of `delivery` in `order` from `start` on. + fn deliveryEnd(parts: []const Part, order: []const usize, start: usize, delivery: Destination.Delivery) usize { + var end = start; + while (end < order.len and parts[order[end]].destination.delivery() == delivery) : (end += 1) {} + return end; + } + + /// The jobs and transfers of one plan as `preparePlan` cuts them. + const JobCut = struct { + /// One part's bytes in a read job: the logical range `[start, end)` + /// of its destination, from `file_offset` of the file on. + const Segment = struct { destination: *Destination, start: u64, end: u64, file_offset: u64 }; + /// A segment's bytes on box `box` of its destination: `len` from + /// `box_offset`, and where a staged job lands them. + const Entry = struct { segment: u32, box: u32, group: u32 = 0, box_offset: u64, len: u64, landing: u64 = 0 }; + /// What a staged job lands together. + const Group = struct { destination: *Destination, box: u32 }; + + allocator: std.mem.Allocator, + parts: []Part, + /// The file every job of the plan reads. + source_slot: *SourceSlot, + config: Config, + reads: Reads, + /// The alignment of the plan's file (`Reads.alignmentOf`). + alignment: usize, + maximum_job_len: usize, + /// The longest logical range of a gather job. + gather_len: usize, + jobs: std.ArrayList(Plan.Job) = .empty, + transfers: std.ArrayList(Plan.Transfer) = .empty, + items: std.ArrayList(Plan.Item) = .empty, + landings: std.ArrayList(u64) = .empty, + extents: std.ArrayList(Plan.Extent) = .empty, + safe_boundaries: std.ArrayList(u64) = .empty, + /// A read job's segments before they are placed, and the staging + /// scratch. + segments: std.ArrayList(Segment) = .empty, + entries: std.ArrayList(Entry) = .empty, + groups: std.AutoHashMapUnmanaged(Group, u32) = .empty, + source_bytes: u64 = 0, + + /// The plan of the jobs cut, copied into `arena` with every context + /// its jobs can need, allocated before publication makes it visible + /// to workers: a request per job and an event per writer of a DMA + /// job's transfer. + fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { + const jobs = self.jobs.items; + const transfers = self.transfers.items; + var dma_submissions: usize = 0; + for (jobs) |job| switch (job.kind) { + .dma_read, .dma_land, .dma_fill => dma_submissions += submissionCount(transfers[job.transfer_start..][0..job.transfer_len]), + .host_direct, .host_bounce, .host_fill => {}, + }; + const requests = try arena.alloc(ReadRequest, jobs.len); + @memset(requests, ReadRequest.idle); + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = self.source_slot, + .jobs = try arena.dupe(Plan.Job, jobs), + .transfers = try arena.dupe(Plan.Transfer, transfers), + .extents = try arena.dupe(Plan.Extent, self.extents.items), + .items = try arena.dupe(Plan.Item, self.items.items), + .landings = try arena.dupe(u64, self.landings.items), + .requests = requests, + .events = try arena.alloc(Dma.EventContext, dma_submissions), + .source_bytes = self.source_bytes, + }; + return plan; + } + + fn deinit(self: *JobCut) void { + const allocator = self.allocator; + self.groups.deinit(allocator); + self.entries.deinit(allocator); + self.segments.deinit(allocator); + self.safe_boundaries.deinit(allocator); + self.extents.deinit(allocator); + self.landings.deinit(allocator); + self.items.deinit(allocator); + self.transfers.deinit(allocator); + self.jobs.deinit(allocator); + } + + /// Cuts the runs of `order` (part indices of one delivery class in + /// file order) into jobs: each run of touching or overlapping + /// ranges into the minimum number of jobs, cut at part-safe + /// boundaries where that number allows. No job depends on another, + /// since every DMA piece is submitted as soon as its block is read. + /// With an alignment (`Reads.alignmentOf`), a job's read widens to + /// aligned bounds around its range (`maximumJobLen` leaves room for + /// it), and the transfers address the widened read. A DMA job's + /// segments are placed in its blocks by `placeDma`. A host job is + /// `host_direct` only when its read is exact and its transfers + /// cover it in order (`coversReadInOrder`), otherwise `host_bounce`. + fn cutRuns(self: *JobCut, order: []const usize, host: bool) !void { + const allocator = self.allocator; + const parts = self.parts; + const alignment = self.alignment; + const maximum_job_len = self.maximum_job_len; + var run_cursor: usize = 0; + while (run_cursor < order.len) { + self.safe_boundaries.clearRetainingCapacity(); + const first_range = parts[order[run_cursor]].fileRange(); + std.debug.assert(first_range.end > first_range.start); // backend.LoadSpec: parts are non-empty + const first_offset = first_range.start; + var run_end = first_range.end; + var run_part_end = run_cursor + 1; + while (run_part_end < order.len) : (run_part_end += 1) { + const candidate = parts[order[run_part_end]].fileRange(); + if (candidate.start > run_end) break; + // A touching range starts where every preceding range has + // ended. Unlike an arbitrary part end, this is safe even + // when the batch contains overlapping or duplicate ranges. + if (candidate.start == run_end and + (self.safe_boundaries.items.len == 0 or + self.safe_boundaries.items[self.safe_boundaries.items.len - 1] != candidate.start)) + { + try self.safe_boundaries.append(allocator, candidate.start); + } + run_end = @max(run_end, candidate.end); + } + + var job_start = first_offset; + var candidate_start = run_cursor; + var boundary_cursor: usize = 0; + const run_len = run_end - first_offset; + const maximum_job_len_u64: u64 = @intCast(maximum_job_len); + var jobs_remaining: usize = @intCast(run_len / maximum_job_len_u64 + + @intFromBool(run_len % maximum_job_len_u64 != 0)); + while (jobs_remaining != 0) { + const safe_boundaries = self.safe_boundaries.items; + const hard_end = @min( + run_end, + std.math.add(u64, job_start, maximum_job_len) catch run_end, + ); + const job_end = if (jobs_remaining == 1) + run_end + else boundary: { + // Less than the run's length, which is a u64. + const remaining_capacity = @as(u64, @intCast(jobs_remaining - 1)) * maximum_job_len_u64; + const minimum_end = @max(job_start + 1, run_end - remaining_capacity); + const maximum_end = @min( + hard_end, + run_end - @as(u64, @intCast(jobs_remaining - 1)), + ); + while (boundary_cursor < safe_boundaries.len and + safe_boundaries[boundary_cursor] <= job_start) + { + boundary_cursor += 1; + } + var scan = boundary_cursor; + var selected: ?u64 = null; + while (scan < safe_boundaries.len and + safe_boundaries[scan] <= maximum_end) : (scan += 1) + { + if (safe_boundaries[scan] >= minimum_end) + selected = safe_boundaries[scan]; + } + boundary_cursor = scan; + break :boundary selected orelse maximum_end; + }; + const job_len: usize = @intCast(job_end - job_start); + const read_start = if (alignment == 0) job_start else std.mem.alignBackward(u64, job_start, alignment); + const read_end = if (alignment == 0) job_end else std.mem.alignForward(u64, job_end, alignment); + const read_len: usize = @intCast(read_end - read_start); + const transfer_start = self.transfers.items.len; + // The job's segments first: where they land in its + // blocks depends on all of them. + self.segments.clearRetainingCapacity(); + while (candidate_start < run_part_end) { + const candidate = parts[order[candidate_start]].fileRange(); + if (candidate.end > job_start) break; + candidate_start += 1; + } + for (order[candidate_start..run_part_end]) |part_index| { + const part = &parts[part_index]; + const range = part.fileRange(); + if (range.start >= job_end) break; + const intersection_start = @max(job_start, range.start); + const intersection_end = @min(job_end, range.end); + if (intersection_start >= intersection_end) continue; + const logical_start = part.lattice.dst_offset + (intersection_start - range.start); + try self.segments.append(allocator, .{ + .destination = part.destination, + .start = logical_start, + .end = logical_start + (intersection_end - intersection_start), + .file_offset = intersection_start, + }); + } + std.debug.assert(self.segments.items.len != 0); + const read: Plan.Job.Read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) }; + const kind: Plan.Job.Kind = if (host) host: { + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + break :host if (alignment == 0 and coversReadInOrder(self.transfers.items[transfer_start..], read_len)) + .{ .host_direct = read } + else + .{ .host_bounce = read }; + } else try self.placeDma(read, read_start, job_start, job_len); + try self.jobs.append(allocator, .{ + .file_offset = read_start, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = kind, + }); + self.source_bytes +|= @intCast(job_len); + job_start = job_end; + jobs_remaining -= 1; + } + std.debug.assert(job_start == run_end); + run_cursor = run_part_end; + } + } + + /// Places a DMA job's segments (`segments`, in file order) in its + /// blocks. Blocks holding the read in file order give one transfer + /// per run (`ShardGrid.run`): a tensor sharded along any axis but + /// its first takes one per row and device, as a column-sharded + /// `o_proj` or `down_proj` does. A staged job lands its read grouped + /// by destination box instead (`stage`), so a box takes one piece + /// per block. Llama-3.1-8B sharded like that on four B70, page + /// cache warm, planned 1.05M submissions and loaded in 5.1 s (~1.2 + /// ms per piece, 64 in flight per device); staged, 5,468 and 0.71 + /// s, against 0.67 s sharded by rows. Staging costs a read + /// scattered over the runs, or for a direct read a copy through the + /// worker's bounce buffer (`landJob`), so a job is staged only where + /// it saves `Config.minimum_staging_gain` submissions, and only when + /// its segments tile its bytes once: overlapping parts send a source + /// byte to two segments, and one read cannot land it twice. + fn placeDma(self: *JobCut, read: Plan.Job.Read, read_start: u64, data_start: u64, job_len: usize) !Plan.Job.Kind { + const block_size = self.config.block_size; + const transfer_start = self.transfers.items.len; + // File order takes a submission per writer of every run, cut by + // blocks, less where runs of adjacent segments continue each other + // or a run ends where a block does: at most `slack` fewer. + var in_file_order: usize = 0; + var slack: usize = 0; + var tiles = true; + var next = data_start; + for (self.segments.items) |segment| { + if (segment.file_offset != next) tiles = false; + next = segment.file_offset + (segment.end - segment.start); + const grid = &segment.destination.grid; + const landing = segment.file_offset - read_start; + const crossings = (landing + segment.end - segment.start - 1) / block_size - landing / block_size; + in_file_order += (grid.runCount(segment.start, segment.end) + crossings) * grid.replicas(); + slack += (1 + crossings) * grid.replicas(); + } + if (tiles and next == data_start + job_len) { + if (try self.stage(read, read_start, job_len, in_file_order, slack)) |land| return .{ .dma_land = land }; + } + for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); + return .{ .dma_read = read }; + } + + /// Lands the job's segments box by box: their bytes on each box of a + /// destination, grouped by (destination, box) in order of first + /// appearance, each group in box order. The job's read is its one + /// extent, and one `Item` per segment tells the worker where its bytes + /// go (`landJob`). Null, with nothing appended, when that saves + /// fewer than `Config.minimum_staging_gain` submissions against file + /// order, estimated at `in_file_order` and at most `slack` fewer: + /// counted exactly, by listing the runs, only when that decides. + fn stage(self: *JobCut, read: Plan.Job.Read, read_start: u64, job_len: usize, in_file_order: usize, slack: usize) !?Plan.Job.Land { + const allocator = self.allocator; + const segments = self.segments.items; + self.entries.clearRetainingCapacity(); + self.groups.clearRetainingCapacity(); + for (segments, 0..) |segment, index| { + const grid = &segment.destination.grid; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, segment.start); + const high = grid.rankIn(box, segment.end); + if (high == low) continue; + const group = try self.groups.getOrPut(allocator, .{ .destination = segment.destination, .box = @intCast(box) }); + if (!group.found_existing) group.value_ptr.* = @intCast(self.groups.count() - 1); + try self.entries.append(allocator, .{ .segment = @intCast(index), .box = @intCast(box), .group = group.value_ptr.*, .box_offset = low, .len = high - low }); + } + } + const entries = self.entries.items; + std.mem.sort(Entry, entries, {}, struct { + fn lessThan(_: void, lhs: Entry, rhs: Entry) bool { + if (lhs.group != rhs.group) return lhs.group < rhs.group; + if (lhs.box_offset != rhs.box_offset) return lhs.box_offset < rhs.box_offset; + return lhs.segment < rhs.segment; + } + }.lessThan); + // The read's own order when every box of a segment is one run and + // the groups come in the order of the runs. + var runs: u64 = 0; + for (segments) |segment| runs += segment.destination.grid.runCount(segment.start, segment.end); + if (runs == entries.len) { + const in_order = for (entries[1..], entries[0 .. entries.len - 1]) |entry, previous| { + if (entry.segment < previous.segment or (entry.segment == previous.segment and entry.box < previous.box)) break false; + } else true; + if (in_order) return null; + } + const staged_start = self.transfers.items.len; + var landing: u64 = 0; + for (entries) |*entry| { + const destination = segments[entry.segment].destination; + entry.landing = landing; + try self.appendBlocked(staged_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = destination.grid.writers[entry.box], + .destination_offset = @intCast(entry.box_offset), + .len = @intCast(entry.len), + }); + landing += entry.len; + } + std.debug.assert(landing == job_len); + const needed = submissionCount(self.transfers.items[staged_start..]) +| self.config.minimum_staging_gain; + var file_order_wins = needed > in_file_order; + if (!file_order_wins and needed +| slack > in_file_order) { + const exact_start = self.transfers.items.len; + for (segments) |segment| try self.appendRunTransfers(exact_start, segment, read_start); + file_order_wins = needed > submissionCount(self.transfers.items[exact_start..]); + self.transfers.shrinkRetainingCapacity(exact_start); + } + if (file_order_wins) { + self.transfers.shrinkRetainingCapacity(staged_start); + return null; + } + + // A landing table per segment, one entry per box of its + // destination. + const item_start = self.items.items.len; + for (segments) |segment| { + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, segment.destination.grid.box_count); + try self.items.append(allocator, .{ + .destination = segment.destination, + .lattice = .{ .src_offset = segment.file_offset, .dst_offset = segment.start, .width = segment.end - segment.start }, + .clip_start = segment.start, + .clip_end = segment.end, + .landing_start = landing_start, + }); + } + for (entries) |entry| { + const item = self.items.items[item_start + entry.segment]; + self.landings.items[item.landing_start + entry.box] = entry.landing -% entry.box_offset; + } + // Only a direct read has padding around its tensor bytes, which + // goes through the worker's bounce buffer. + const extent_start = self.extents.items.len; + try self.extents.append(allocator, .{ + .source_slot = self.source_slot, + .file_offset = read_start, + .len = read.len, + .minimum_len = read.minimum_len, + .scatter = self.alignment == 0, + .item_start = item_start, + .item_len = segments.len, + }); + return .{ .len = job_len, .extent_start = extent_start, .extent_len = 1 }; + } + + /// Appends `segment`'s transfers with its bytes where the read holds + /// them, one per run (`ShardGrid.run`) and block, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it: parts adjacent in the file and in + /// the destination go out as one transfer. + fn appendRunTransfers(self: *JobCut, transfer_start: usize, segment: Segment, read_start: u64) !void { + const destination = segment.destination; + const grid = &destination.grid; + var at = segment.start; + while (at < segment.end) { + const stop = grid.runEnd(at, segment.end); + const box, const box_offset = grid.locate(at); + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(segment.file_offset + (at - segment.start) - read_start), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(box_offset), + .len = @intCast(stop - at), + }); + at = stop; + } + } + + /// Plans a gathered destination (`Destination.gathered`) whole: its + /// logical bytes cut into ranges of at most `gather_len`, each one + /// job. For every part with runs in the range the job reads the + /// source range those runs come from (an `Extent`, from whichever + /// file holds it), and it lands the range in its blocks box by box, + /// one transfer per box and block, its bytes no part covers zero. + /// A part whose source runs are out of order (a broadcast repeats + /// them, a transpose permutes them) reads its whole source range + /// in every job (`stridedRead`). An interleave of members reads + /// each member's rows of the range, so the range lands in one piece + /// per box and block whatever the interleave's granularity. + fn appendGatherJobs(self: *JobCut, destination: *Destination) !void { + const allocator = self.allocator; + const grid = &destination.grid; + const parts = destination.parts; + // reach[i]: the furthest destination byte parts[0..=i] reach, so + // a range's first candidate is found by bisection. + const reach = try allocator.alloc(u64, parts.len); + defer allocator.free(reach); + var furthest: u64 = 0; + for (parts, reach) |*part, *at| { + const spans = part.lattice.spans().?; + furthest = @max(furthest, part.lattice.dst_offset + spans.dst); + at.* = furthest; + } + var start: u64 = 0; + while (start < grid.total) { + const end = @min(grid.total, start + self.gather_len); + const transfer_start = self.transfers.items.len; + const extent_start = self.extents.items.len; + const landing_start = self.landings.items.len; + try self.landings.appendNTimes(allocator, 0, grid.box_count); + var landing: u64 = 0; + for (0..grid.box_count) |box| { + const low = grid.rankIn(box, start); + const high = grid.rankIn(box, end); + if (high == low) continue; + self.landings.items[landing_start + box] = landing -% low; + try self.appendBlocked(transfer_start, .{ + .destination = destination, + .landing = @intCast(landing), + .writer_mask = grid.writers[box], + .destination_offset = @intCast(low), + .len = @intCast(high - low), + }); + landing += high - low; + } + std.debug.assert(landing == end - start); + + var covered: u64 = 0; + var index = std.sort.partitionPoint(u64, reach, start, struct { + fn below(limit: u64, at: u64) bool { + return at <= limit; + } + }.below); + while (index < parts.len and parts[index].lattice.dst_offset < end) : (index += 1) { + const part = &parts[index]; + const first = part.lattice.firstEndingAfter(start); + const last = part.lattice.firstStartingFrom(end); + if (first >= last) continue; + covered += part.lattice.coveredIn(start, end); + try self.appendExtent(part, first, last, start, end, landing_start); + } + std.debug.assert(covered <= end - start); + const job_len: usize = @intCast(end - start); + const extent_len = self.extents.items.len - extent_start; + try self.jobs.append(allocator, .{ + .file_offset = if (extent_len != 0) self.extents.items[extent_start].file_offset else 0, + .transfer_start = transfer_start, + .transfer_len = self.transfers.items.len - transfer_start, + .kind = .{ .dma_land = .{ + .len = job_len, + .extent_start = extent_start, + .extent_len = extent_len, + .zeroed = covered < end - start, + } }, + }); + start = end; + } + } + + /// The extent reading `part`'s runs `first` to `last` (exclusive) + /// clipped to `[start, end)`, with the one item that places them: + /// from the first run's first byte in the range to the last run's + /// last one when the part's source runs are in order, else the + /// part's whole source range. + fn appendExtent(self: *JobCut, part: *const Part, first: u64, last: u64, start: u64, end: u64, landing_start: usize) !void { + const lattice = part.lattice; + const width = lattice.width; + var source_start = lattice.src_offset; + var source_end = lattice.src_offset + lattice.spans().?.src; + const ordered = lattice.srcOrdered(); + if (ordered) { + const head = lattice.at(first); + const tail = lattice.at(last - 1); + source_start = head.src + (start -| head.dst); + source_end = tail.src + width - ((tail.dst + width) -| end); + } + const file_start = part.source.offset + source_start; + const file_end = part.source.offset + source_end; + const alignment = try self.reads.alignmentOf(part.source_slot); + const read_start = if (alignment == 0) file_start else std.mem.alignBackward(u64, file_start, alignment); + const read_end = if (alignment == 0) file_end else std.mem.alignForward(u64, file_end, alignment); + // The range of a part whose runs are back to back is the job's; + // any other part's whole range fits (`stridedRead`). + std.debug.assert(read_end - read_start <= self.config.request_size); + try self.extents.append(self.allocator, .{ + .source_slot = part.source_slot, + .file_offset = read_start, + .len = @intCast(read_end - read_start), + .minimum_len = @intCast(file_end - read_start), + .scatter = alignment == 0 and ordered and lattice.srcTiled(), + .item_start = self.items.items.len, + .item_len = 1, + }); + try self.items.append(self.allocator, .{ + .destination = part.destination, + .lattice = .{ .src_offset = part.source.offset + lattice.src_offset, .dst_offset = lattice.dst_offset, .width = width, .dims = lattice.dims }, + .first = first, + .end = last, + .clip_start = start, + .clip_end = end, + .landing_start = landing_start, + }); + self.source_bytes +|= file_end - file_start; + } + + /// Appends `transfer`, cut at block boundaries, merged with the + /// previous transfer from `transfer_start` on where block, writers + /// and both offsets continue it. + fn appendBlocked(self: *JobCut, transfer_start: usize, transfer: Plan.Transfer) !void { + const block_size = self.config.block_size; + var cursor: usize = 0; + while (cursor < transfer.len) { + const at = transfer.landing + cursor; + const in_block = at % block_size; + const take = @min(transfer.len - cursor, block_size - in_block); + const destination_offset = transfer.destination_offset + cursor; + cursor += take; + // A transfer that ends where this one starts is in its + // block unless a block starts here. + if (self.transfers.items.len > transfer_start and in_block != 0) { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination == transfer.destination and + previous.writer_mask == transfer.writer_mask and + previous.landing + previous.len == at and + previous.destination_offset + previous.len == destination_offset) + { + previous.len += take; + continue; + } + } + try self.transfers.append(self.allocator, .{ + .destination = transfer.destination, + .landing = at, + .writer_mask = transfer.writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + } + } + + fn submissionCount(transfers: []const Plan.Transfer) usize { + var count: usize = 0; + for (transfers) |transfer| count += @popCount(transfer.writer_mask); + return count; + } + + /// The zero fill of the destinations whose first part is among + /// `order`, the file's parts of one delivery class, as one job. A DMA + /// fill job reads one block, the zero block; a host one writes zeros + /// in place. A gathered destination's jobs write its zeros + /// themselves. + fn appendFillJob(self: *JobCut, order: []const usize, host: bool) !void { + const fill_start = self.transfers.items.len; + for (order) |part_index| { + const part = &self.parts[part_index]; + const destination = part.destination; + if (part != &destination.parts[0]) continue; + try self.appendFillTransfers(fill_start, destination); + } + const fill_transfers = self.transfers.items[fill_start..]; + if (fill_transfers.len == 0) return; + try self.jobs.append(self.allocator, .{ + .file_offset = 0, + .transfer_start = fill_start, + .transfer_len = fill_transfers.len, + .kind = if (host) .host_fill else .dma_fill, + }); + } + + /// Appends the zero-fill transfers of `destination`: every logical + /// range no part covers (the gaps between its parts and the tail), + /// placed by its grid like read bytes, each piece at most one block + /// at landing 0, where a DMA job reads the zero block. Pieces count + /// toward their targets' totals, so the fill completes each + /// device's placement bytes alongside the parts. + fn appendFillTransfers(self: *JobCut, transfer_start: usize, destination: *Destination) !void { + var covered_end: u64 = 0; + for (destination.parts) |part| { + const lattice = part.lattice; + if (lattice.dst_offset > covered_end) try self.appendFillRange(transfer_start, destination, covered_end, lattice.dst_offset); + covered_end = lattice.dst_offset + lattice.width; + } + if (covered_end < destination.grid.total) try self.appendFillRange(transfer_start, destination, covered_end, destination.grid.total); + } + + /// The fill of the logical range `[start, end)`: one range of each box + /// it touches, cut by blocks, merged where one writer set's pieces + /// are contiguous. + fn appendFillRange(self: *JobCut, transfer_start: usize, destination: *Destination, start: u64, end: u64) !void { + const block_size = self.config.block_size; + const grid = &destination.grid; + for (0..grid.box_count) |box| { + var destination_offset: usize = @intCast(grid.rankIn(box, start)); + var remaining: usize = @intCast(grid.rankIn(box, end) - destination_offset); + const writer_mask = grid.writers[box]; + while (remaining != 0) { + if (self.transfers.items.len > transfer_start) merge: { + const previous = &self.transfers.items[self.transfers.items.len - 1]; + if (previous.destination != destination or previous.writer_mask != writer_mask or + previous.destination_offset + previous.len != destination_offset or + previous.len == block_size) + break :merge; + const grown = @min(remaining, block_size - previous.len); + previous.len += grown; + destination_offset += grown; + remaining -= grown; + continue; + } + const take = @min(remaining, block_size); + try self.transfers.append(self.allocator, .{ + .destination = destination, + .landing = 0, + .writer_mask = writer_mask, + .destination_offset = destination_offset, + .len = take, + }); + destination_offset += take; + remaining -= take; + } + } + } + + /// Whether a host job's transfers address its read from byte zero + /// to `read_len` in order, each source byte once: then the read + /// can scatter straight into the destinations. Overlapping or + /// duplicate parts address some source bytes twice and go through + /// blocks instead. + fn coversReadInOrder(transfers: []const Plan.Transfer, read_len: usize) bool { + var next: usize = 0; + for (transfers) |transfer| { + if (transfer.landing != next) return false; + next += transfer.len; + } + return next == read_len; + } + }; + fn blockCount(len: usize, block_size: usize) usize { + return len / block_size + @intFromBool(len % block_size != 0); + } + + /// The source bytes every job of `part` reads whole, against one read + /// (`Config.gather_limit`, at most a request). A part whose source runs + /// are back to back reads only its job's range: 0. Any other counts its + /// whole source range, widened to the widest alignment its file may be + /// read at (`Reads.boundOf`): out of order (a broadcast repeats its + /// runs, a transpose permutes them), every job reads that range + /// (`JobCut.appendExtent`); in order with gaps (a slice along an inner + /// axis), a job reads from its first run to its last, which the gaps + /// stretch past any job's length. + fn stridedRead(config: Config, reads: Reads, part: LoadPart) backend.Backend.StridedRead { + const lattice = part.lattice; + const limit = @min(config.request_size, config.gather_limit); + if (lattice.srcTiled()) return .{ .read = 0, .limit = limit }; + const alignment = @max(reads.boundOf(part.source.file_uri), 1); + const start = part.source.offset + lattice.src_offset; + const end = start + lattice.spans().?.src; + return .{ + .read = std.mem.alignForward(u64, end, alignment) - std.mem.alignBackward(u64, start, alignment), + .limit = limit, + }; + } + + /// The longest tensor range one job may cover: the request size, less + /// the two alignment units its widened read can add, so a widened job + /// still fits a request (a credit's slot). `backend.requestSize` refuses + /// a request that cannot hold them. + fn maximumJobLen(request_size: usize, alignment: usize) usize { + std.debug.assert(request_size != 0); + if (alignment == 0) return request_size; + std.debug.assert(std.math.isPowerOfTwo(alignment) and 2 * alignment < request_size); + return request_size - 2 * alignment; + } + + /// Part indices sorted by file URI, delivery class + /// (`Destination.Delivery`), offset, size and index: the planner's + /// input, taken one file group at a time (`fileGroupEnd`), each class + /// in file order. + fn sortedPartOrder(allocator: std.mem.Allocator, parts: []const Part) ![]usize { + const order = try allocator.alloc(usize, parts.len); + for (order, 0..) |*index, i| index.* = i; + const SortContext = struct { + parts: []const Part, + + fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { + const left = ctx.parts[lhs]; + const right = ctx.parts[rhs]; + const uri_order = std.mem.order(u8, left.source.file_uri, right.source.file_uri); + if (uri_order != .eq) return uri_order == .lt; + const left_delivery = left.destination.delivery(); + const right_delivery = right.destination.delivery(); + if (left_delivery != right_delivery) return @backingInt(left_delivery) < @backingInt(right_delivery); + const left_offset = left.fileRange().start; + const right_offset = right.fileRange().start; + if (left_offset != right_offset) return left_offset < right_offset; + if (left.lattice.width != right.lattice.width) return left.lattice.width < right.lattice.width; + return lhs < rhs; + } + }; + std.mem.sort(usize, order, SortContext{ .parts = parts }, SortContext.lessThan); + return order; + } + + /// The end of the file group that begins at `order[start]`. + fn fileGroupEnd(parts: []const Part, order: []const usize, start: usize) usize { + const uri = parts[order[start]].source.file_uri; + var end = start + 1; + while (end < order.len and std.mem.eql(u8, uri, parts[order[end]].source.file_uri)) : (end += 1) {} + return end; + } +}; + +/// A strict FIFO of published plans. A submission holds one plan per source +/// file, published as soon as that file is planned, so a plan reaches the +/// queue behind every earlier plan of its own submission and of every +/// earlier one (submissions publish and seal on one task). Within a plan, +/// jobs are handed out in the planned order (file order); a plan leaves the +/// queue with its last claim or through `fail`. +/// A plan is queued only with jobs left and its completion units are added +/// in the same critical section, so a queued plan's batch still holds at +/// least one unit and can never be freed under the scheduler. +const Scheduler = struct { + /// A claimed job: its position in the plan that owns it. The claim holds + /// one of the batch's completion units until the request releases it. + const Claim = struct { + plan: *Plan, + index: usize, + }; + + allocator: std.mem.Allocator, + /// Plans with unclaimed jobs in publish order. + queue: std.Deque(*Plan) = .empty, + /// Set by `fail`: `publish` refuses plans (`Canceled`) and `waitForWork` + /// returns false. + stopping: bool = false, + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator) Scheduler { + return .{ .allocator = allocator }; + } + + fn deinit(self: *Scheduler) void { + // Every batch was awaited (and so claimed out or retired) first. + std.debug.assert(self.queue.len == 0); + self.queue.deinit(self.allocator); + self.* = undefined; + } + + /// Publishes one plan of an open submission behind every earlier plan. + /// A plan without jobs never joins the queue. `Canceled` once the + /// scheduler failed (`fail`). + fn publish(self: *Scheduler, io: std.Io, batch: *Batch, plan: *Plan) !void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + if (self.stopping) return error.Canceled; + const job_count = plan.jobs.len; + try batch.plans.ensureUnusedCapacity(batch.arena.allocator(), 1); + if (job_count != 0) try self.queue.ensureUnusedCapacity(self.allocator, 1); + // Nothing below fails: the plan, its units and the queue entry + // appear together. + plan.batch = batch; + batch.plans.appendAssumeCapacity(plan); + _ = batch.remaining.fetchAdd(job_count, .acq_rel); + if (job_count != 0) { + self.queue.pushBackAssumeCapacity(plan); + self.condition.broadcast(io); + } + } + + /// Stops claims and publishes, wakes every worker waiting for work, and + /// retires the unclaimed units of every queued plan so its batch still + /// reaches `done` through its claimed requests (an open one through its + /// seal and sentinel as well). Claims and this pass both move cursors + /// under the mutex, so they partition the jobs exactly: every claimed + /// job keeps its unit with its worker. Each queued plan still holds an + /// unclaimed unit of its batch, so only the retire of a batch's last + /// queued plan can complete it, and nothing touches the batch after that. + fn fail(self: *Scheduler, io: std.Io) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + self.stopping = true; + self.condition.broadcast(io); + while (self.queue.popFront()) |plan| { + const unclaimed = plan.jobs.len - plan.cursor; + plan.cursor = plan.jobs.len; + plan.batch.releaseUnits(unclaimed); + } + } + + /// Hands out the head plan's next job; the plan leaves the queue with + /// its last one. Null only when nothing is queued: a queued plan always + /// has a job left. + fn claim(self: *Scheduler, io: std.Io) ?Claim { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const plan = self.queue.front() orelse return null; + const index = plan.cursor; + std.debug.assert(index < plan.jobs.len); + plan.cursor += 1; + if (plan.cursor == plan.jobs.len) _ = self.queue.popFront(); + return .{ .plan = plan, .index = index }; + } + + fn waitForWork(self: *Scheduler, io: std.Io) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (!self.stopping and self.queue.len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + return !self.stopping; + } +}; + +/// A claimed source job and its completion state. The plan owns its storage. +const ReadRequest = struct { + const Scratch = struct { + allocator: std.mem.Allocator, + /// A read's ranges: one per block of a job (`readBlocks`), or a + /// scattered read's, at most one vectored read's worth (`Scatter`). + iovecs: [][]u8, + /// A landing job's reads that do not scatter (`landJob`), allocated + /// on first use: as large as a job's blocks, page aligned for + /// O_DIRECT. + bounce: []align(std.heap.page_size_min) u8 = &.{}, + bounce_capacity: usize, + + fn init(allocator: std.mem.Allocator, maximum_blocks: usize, block_size: usize) !Scratch { + return .{ + .allocator = allocator, + .iovecs = try allocator.alloc([]u8, @max(maximum_blocks, max_positional_iovecs)), + .bounce_capacity = maximum_blocks * block_size, + }; + } + + /// The ranges a `Scatter` gathers before it reads. + fn scatterIovecs(self: *const Scratch) [][]u8 { + return self.iovecs[0..max_positional_iovecs]; + } + + fn bounceBuffer(self: *Scratch, len: usize) ![]u8 { + std.debug.assert(len <= self.bounce_capacity); + if (self.bounce.len == 0) self.bounce = try self.allocator.alignedAlloc(u8, .fromByteUnits(std.heap.page_size_min), self.bounce_capacity); + return self.bounce[0..len]; + } + + fn deinit(self: *Scratch) void { + self.allocator.free(self.bounce); + self.allocator.free(self.iovecs); + self.* = undefined; + } + }; + + loader: *Loader, + plan: *Plan, + /// The job's index in its plan, which holds this request at the same + /// index. + index: usize, + /// The request's lifecycle credit: its slot of blocks (`Credits`), which + /// only this request reads into and transfers from until it ends. + slot: u32, + /// The scheduling sentinel and one reference per DMA piece queued; the + /// last release gives the credit back and finishes the job. + pending: std.atomic.Value(usize) = .init(1), + /// Awake-clock nanoseconds of the enqueue; 0 until then. + enqueued_ns: u64 = 0, + + /// The slot of a job that was never claimed: nothing pending, so the + /// retirement checks hold. + const idle: ReadRequest = .{ + .loader = undefined, + .plan = undefined, + .index = undefined, + .slot = undefined, + .pending = .init(0), + }; + + /// Takes the claimed job's request context with the worker's credit. + /// The claim holds the batch's completion unit; the request's final + /// reference drop releases it. + fn init(loader: *Loader, claim: Scheduler.Claim, slot: u32) *ReadRequest { + const self = &claim.plan.requests[claim.index]; + self.* = .{ + .loader = loader, + .plan = claim.plan, + .index = claim.index, + .slot = slot, + }; + return self; + } + + fn run(self: *ReadRequest, scratch: *Scratch) !void { + const loader = self.loader; + const io = loader.io; + const block_size = loader.plan_config.block_size; + const job = self.plan.jobs[self.index]; + const transfers = self.plan.transfersOf(job); + // The first worker to touch a destination creates its state; a + // host-resident one's is normally created ahead by the allocation + // task. + var previous: ?*Destination = null; + for (transfers) |transfer| { + const destination = transfer.destination; + if (destination == previous) continue; + previous = destination; + const init_started = awakeNs(io); + _ = destination.ensureState(loader) catch |err| { + loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); + return err; + }; + _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(io) -| init_started, .monotonic); + } + const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; + switch (job.kind) { + .dma_read => |read| try self.readBlocks(scratch, blocks, read), + .dma_land => |land| try landJob(loader, self.plan, land, blocks, scratch), + // Reads nothing: its transfers read the zero block. + .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), + // A host job's worker writes its destinations itself, so the job + // ends with the worker's scheduling reference; its slot's blocks, + // if it reads through them, are done when this returns. + .host_direct => |read| { + var scatter: Scatter = .{ + .loader = loader, + .slot = self.plan.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = job.file_offset, + }; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The planner checked that the transfers cover the read + // in order, each source byte once. + try scatter.add(tensor.hostLanding(transfer)); + } + try scatter.flush(); + std.debug.assert(scatter.file_offset == job.file_offset + read.len); + }, + .host_bounce => |read| { + try self.readBlocks(scratch, blocks, read); + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + // The last block holds stale bytes past the read, which + // no bounds check would stop a copy from taking. + std.debug.assert(transfer.landing + transfer.len <= read.len); + const piece = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(piece.len == transfer.len); + @memcpy(tensor.hostLanding(transfer), piece); + } + }, + .host_fill => { + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + @memset(tensor.hostLanding(transfer), 0); + } + return; + }, + } + for (transfers) |transfer| transfer.destination.recordRead(transfer.len); + if (job.kind == .dma_read or job.kind == .dma_land) try Dma.enqueue(self, blocks); + } + + /// Reads the job's `read` into `blocks`, its slot's, in file order. + fn readBlocks(self: *ReadRequest, scratch: *Scratch, blocks: []const host_memory.Block, read: Plan.Job.Read) !void { + const loader = self.loader; + const block_size = loader.plan_config.block_size; + const iovecs = scratch.iovecs[0..blocks.len]; + for (iovecs, blocks, 0..) |*iovec, block, block_index| { + iovec.* = block[0..@min(block_size, read.len - block_index * block_size)]; + } + try readSlot(loader, self.plan.source_slot, iovecs, self.plan.jobs[self.index].file_offset, read.minimum_len); + } + + /// Releases a DMA piece or the scheduling reference. The final release + /// gives the credit back and may let the awaiting task free this request + /// and everything its batch owns. + fn release(self: *ReadRequest) void { + const previous = self.pending.fetchSub(1, .acq_rel); + std.debug.assert(previous > 0); + if (previous != 1) return; + + // Locals first: releasing the batch unit may complete the batch + // that owns this request, so nothing is touched after it. + const loader = self.loader; + const batch = self.plan.batch; + if (self.enqueued_ns != 0) { + _ = loader.metrics.dma_stage_ns.fetchAdd(awakeNs(loader.io) -| self.enqueued_ns, .monotonic); + } + loader.credits.release(loader.io, self.slot); + batch.releaseUnits(1); + } +}; + +/// Reads `[file_offset, file_offset + minimum)`, and possibly more, into +/// `buffers` in order, consuming them as it reads; bytes past `minimum` may +/// stay unread. The io takes as many buffers per call as it can (VFS's +/// direct read `IOV_MAX`, std's and zio's 8, a remote read all); a short +/// read resumes where it stopped. Reading stops once `minimum` bytes are +/// in, so the file may end only past `minimum`: the bytes after it are +/// padding a caller added to align the read, and the read that would fetch +/// them can start unaligned, which a file read directly refuses. Returns +/// the bytes read. +fn readFilePositionalAllV( + io: std.Io, + file: std.Io.File, + buffers: [][]u8, + file_offset: u64, + minimum: u64, +) !u64 { + var index: usize = 0; + var completed: u64 = 0; + while (completed < minimum) { + std.debug.assert(index < buffers.len); + const bytes_read = try file.readPositional(io, buffers[index..], file_offset + completed); + if (bytes_read == 0) return error.UnexpectedEndOfFile; + completed += bytes_read; + var rest = bytes_read; + while (index < buffers.len and rest >= buffers[index].len) : (index += 1) rest -= buffers[index].len; + if (rest != 0) buffers[index] = buffers[index][rest..]; + } + return completed; +} + +/// Every worker read goes through here, under a read permit: a failure is +/// recorded with the file, since a landing job's extents can come from files +/// other than its plan's. Once the loader failed, returns its first error. +/// Consumes `iovecs` (`readFilePositionalAllV`). +fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64, minimum_len: usize) !void { + const io = loader.io; + const file = slot.ensure(io) catch |err| { + loader.recordError(err, "open {s}", .{slot.uri}); + return err; + }; + loader.read_permits.waitUncancelable(io); + defer loader.read_permits.post(io); + if (loader.first_error.get()) |err| return err; + const read_started = awakeNs(io); + const read_result = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len); + _ = loader.metrics.read_ns.fetchAdd(awakeNs(io) -| read_started, .monotonic); + const bytes_read = read_result catch |err| { + loader.recordError(err, "read {s} at {d}, {d} bytes at least", .{ slot.uri, file_offset, minimum_len }); + return err; + }; + _ = loader.metrics.read_operations.fetchAdd(1, .monotonic); + _ = loader.metrics.read_bytes.fetchAdd(@intCast(bytes_read), .monotonic); +} + +/// Reads a landing job's extents and lands their items' runs in its blocks +/// (`Plan.Item`, `walkItem`). An extent marked for it scatters +/// straight into the blocks, `max_positional_iovecs` ranges per call; any +/// other lands in the worker's bounce buffer and is copied from there: a +/// direct read, whose ranges must be aligned, and a gather read whose +/// source runs are out of order or repeat (a transpose, a broadcast). +fn landJob( + loader: *Loader, + plan: *const Plan, + land: Plan.Job.Land, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + if (land.zeroed) { + for (blocks) |block| @memset(block[0..loader.plan_config.block_size], 0); + } + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| { + try landExtent(loader, extent, plan.items[extent.item_start..][0..extent.item_len], plan.landings, blocks, scratch); + } +} + +fn landExtent( + loader: *Loader, + extent: Plan.Extent, + items: []const Plan.Item, + landings: []const u64, + blocks: []const host_memory.Block, + scratch: *ReadRequest.Scratch, +) !void { + const block_size = loader.plan_config.block_size; + if (extent.scatter) { + std.debug.assert(extent.len == extent.minimum_len); + var scatter: Scatter = .{ + .loader = loader, + .slot = extent.source_slot, + .iovecs = scratch.scatterIovecs(), + .file_offset = extent.file_offset, + .blocks = blocks, + }; + for (items) |item| try walkItem(item, landings, &scatter); + try scatter.flush(); + std.debug.assert(scatter.file_offset == extent.file_offset + extent.len); + return; + } + const bounce = try scratch.bounceBuffer(extent.len); + var buffers = [_][]u8{bounce}; + try readSlot(loader, extent.source_slot, &buffers, extent.file_offset, extent.minimum_len); + var sink: CopySink = .{ .bounce = bounce, .file_offset = extent.file_offset, .blocks = blocks, .block_size = block_size }; + for (items) |item| try copyItem(item, landings, &sink); +} + +/// Hands `sink` every run of `item` as `place(file offset, landing, len)`, +/// in run order, cut where the destination's box changes +/// (`ShardGrid.runEnd`). +fn walkItem(item: Plan.Item, landings: []const u64, sink: anytype) !void { + const table = landings[item.landing_start..][0..item.destination.grid.box_count]; + try walkRuns(item.lattice, item.first, item.end, item.clip_start, item.clip_end, &item.destination.grid, table, sink); +} + +/// `walkItem` for runs `first` to `end` of `lattice`, clipped to the +/// destination range `[clip_start, clip_end)`. +fn walkRuns(lattice: backend.Lattice, first: u64, end: u64, clip_start: u64, clip_end: u64, grid: *const ShardGrid, table: []const u64, sink: anytype) !void { + if (first >= end) return; + var runs: backend.Lattice.Cursor = .init(lattice, first, end); + while (runs.next()) |run| { + var at = @max(run.dst, clip_start); + const stop = @min(run.dst + lattice.width, clip_end); + var source = run.src + (at - run.dst); + while (at < stop) { + const piece_end = grid.runEnd(at, stop); + const box, const box_offset = grid.locate(at); + try sink.place(source, table[box] +% box_offset, piece_end - at); + source += piece_end - at; + at = piece_end; + } + } +} + +/// `walkItem` from the bounce buffer, a stretch at a time where that pays: +/// a stretch of the destination landing in one piece, inside one run of +/// the grid and one block, has its whole runs copied by +/// `Lattice.copyRuns`, and only the runs its ends cut go one at a time. A +/// stretch costs a few bisections, so items with few runs per stretch (a +/// part of a few wide runs, a grid that cuts every row) are walked. +fn copyItem(item: Plan.Item, landings: []const u64, sink: *CopySink) !void { + const lattice = item.lattice; + const grid = &item.destination.grid; + if (lattice.dims.len == 0 or item.first >= item.end) return walkItem(item, landings, sink); + const start = @max(item.clip_start, lattice.at(item.first).dst); + const stop = @min(item.clip_end, lattice.at(item.end - 1).dst + lattice.width); + if (start >= stop) return; + const stretches = grid.runCount(start, stop) + (stop - start) / sink.block_size + 1; + if (item.end - item.first < 16 * stretches) return walkItem(item, landings, sink); + const table = landings[item.landing_start..][0..grid.box_count]; + const source = @intFromPtr(sink.bounce.ptr) -% @as(usize, @intCast(sink.file_offset)); + var at = start; + while (at < stop) { + const box, const box_offset = grid.locate(at); + const landing: usize = @intCast(table[box] +% box_offset); + const in_block = landing % sink.block_size; + const stretch_end = @min(grid.runEnd(at, stop), at + (sink.block_size - in_block)); + // The address logical offset zero would have, were the whole + // destination laid out like this stretch. + const destination = @intFromPtr(sink.blocks[landing / sink.block_size].ptr) + in_block -% @as(usize, @intCast(at)); + const touched = @max(item.first, lattice.firstEndingAfter(at)); + const whole = @max(item.first, lattice.firstStartingFrom(at)); + const whole_end = @min(item.end, lattice.firstEndingAfter(stretch_end)); + const touched_end = @min(item.end, lattice.firstStartingFrom(stretch_end)); + if (whole < whole_end) { + try walkRuns(lattice, touched, whole, at, stretch_end, grid, table, sink); + lattice.copyRuns(whole, whole_end, destination, source); + try walkRuns(lattice, whole_end, touched_end, at, stretch_end, grid, table, sink); + } else { + try walkRuns(lattice, touched, touched_end, at, stretch_end, grid, table, sink); + } + at = stretch_end; + } +} + +/// The ranges a `Scatter` gathers before it reads, as many as one direct +/// read takes. +const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; + +/// A read scattered over memory ranges as they come, in file order and +/// back to back from `file_offset` of `slot`'s file: a range that continues +/// the previous one in memory merges with it, and one call reads at most +/// `iovecs.len` ranges. Fed destination ranges directly (`add`), or the runs +/// of a walk landing in `blocks` (`place`). Every byte read must exist: a +/// scattered read has no alignment padding. +const Scatter = struct { + loader: *Loader, + slot: *SourceSlot, + iovecs: [][]u8, + count: usize = 0, + /// Bytes the pending ranges cover. + pending: usize = 0, + /// The file offset of the first byte not read yet. + file_offset: u64, + blocks: []const host_memory.Block = &.{}, + + fn add(self: *Scatter, range: []u8) !void { + if (self.count != 0) { + const last = &self.iovecs[self.count - 1]; + if (last.ptr + last.len == range.ptr) { + last.len += range.len; + self.pending += range.len; + return; + } + } + if (self.count == self.iovecs.len) try self.flush(); + self.iovecs[self.count] = range; + self.count += 1; + self.pending += range.len; + } + + /// A run's landing in the blocks, split at block boundaries. + fn place(self: *Scatter, source: u64, landing: u64, len: u64) !void { + std.debug.assert(source == self.file_offset + self.pending); + var at: usize = @intCast(landing); + var rest: usize = @intCast(len); + while (rest != 0) { + const piece = blockPiece(self.blocks, self.loader.plan_config.block_size, at, rest); + try self.add(piece); + at += piece.len; + rest -= piece.len; + } + } + + fn flush(self: *Scatter) !void { + if (self.count == 0) return; + try readSlot(self.loader, self.slot, self.iovecs[0..self.count], self.file_offset, self.pending); + self.file_offset += self.pending; + self.count = 0; + self.pending = 0; + } +}; + +/// Runs copied from a read in the bounce buffer, which starts at +/// `file_offset`. +const CopySink = struct { + bounce: []const u8, + file_offset: u64, + blocks: []const host_memory.Block, + block_size: usize, + + fn place(self: *CopySink, source: u64, landing: u64, len: u64) !void { + var at: usize = @intCast(landing); + var rest = self.bounce[@intCast(source - self.file_offset)..][0..@intCast(len)]; + while (rest.len != 0) { + const piece = blockPiece(self.blocks, self.block_size, at, rest.len); + @memcpy(piece, rest[0..piece.len]); + rest = rest[piece.len..]; + at += piece.len; + } + } +}; + +/// The bytes from offset `at` of `blocks` taken end to end, `block_size` +/// bytes each, up to `len` of them or the end of that block. +fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { + const in_block = at % block_size; + return blocks[at / block_size][in_block..][0..@min(len, block_size - in_block)]; +} + +/// The DMA stage, over the loader's pumps (`Loader.pumps`): workers queue a +/// request's transfers per device, a pump submits them under the device's +/// in-flight budget, and their callbacks release the request. +const Dma = struct { + /// One DMA piece: `data`, in its request's slot, for `target` at + /// `destination_offset`. The piece holds one of its request's + /// references until its callback. Replicas share their source bytes: + /// copying them to another CPU socket first was rejected on MI300X, + /// where raw local and remote H2D both reached ~49-50 GiB/s per GPU + /// while one CPU thread copied across sockets at 5-10 GiB/s. + const Piece = struct { + target: *DestinationState.Target, + request: *ReadRequest, + data: []const u8, + destination_offset: usize, + }; + + /// Pieces ready for one device, submitted in arrival order so the + /// oldest requests and submissions complete first. Owned by the + /// device's `DevicePump.mutex`. + const ReadyQueue = std.Deque(Piece); + + /// One device's submission state. Its pump runs on whichever thread + /// finds work -- a worker after enqueueing, a ready callback after a + /// completion -- one at a time per device (`pumping`), and shares + /// nothing with the other devices' pumps. Per-device pumps load + /// replicated DeepSeek on four GB300 in 4.68 s, against 5.41 s with one + /// pump for every device, at unchanged pinned high-water. A dedicated + /// task per device took 5.0-5.4 s with a wake per completion (a futex + /// round trip per piece), or 4.4-5.1 s with half-budget wake hysteresis. + const DevicePump = struct { + mutex: std.Io.Mutex = .init, + queue: ReadyQueue = .empty, + /// In-flight DMA bytes and submissions; owned by `mutex`. + active_bytes: usize = 0, + active_pieces: usize = 0, + /// True while one thread runs this device's pump; owned by `mutex`. + /// One submitter per device is a correctness requirement: + /// `Target.nextIsLast` flags a tensor's last piece in submission + /// order. + pumping: bool = false, + /// Contexts whose callback fired, for the next pump or a batch's + /// retirement to destroy, never the event's own callback: an + /// intrusive stack through `EventContext.next_retired`, owned by + /// `mutex`. Destroying an event from the next pump instead of at its + /// batch's retirement bounds live PJRT events by the DMA width plus + /// one pump batch rather than by a submission's transfer count; + /// every shipped plugin accepted destruction outside the event's own + /// callback (16,384 fired events on two B70, CUDA and ROCm alike). + retired: ?*EventContext = null, + + /// Whether the next queued piece may go. The budget is in bytes, + /// not submissions: `dma_calibration.depth` blocks, and a piece is + /// one tensor's slice of a block, so a model of small tensors needs + /// many more pieces in flight to keep the same bytes moving: eight + /// average DeepSeek pieces occupy only ~18 MiB against the 128 MiB + /// budget of 16 MiB blocks. The piece that crosses the budget is + /// admitted: a device with room always has a piece in flight. + fn hasRoom(self: *const DevicePump, budget_bytes: usize) bool { + return self.active_bytes < budget_bytes and self.active_pieces < max_dma_pieces_per_device; + } + + /// `mutex`. Destroys every retired event. + fn destroyRetired(self: *DevicePump) void { + while (self.retired) |ctx| { + self.retired = ctx.next_retired; + ctx.next_retired = null; + ctx.destroyEvent(); + } + } + }; + + const EventContext = struct { + request: *ReadRequest, + /// Null once destroyed: by a pump or a batch's retirement after its + /// callback ran, or at once by a failed `onReady`. + pjrt_event: ?*pjrt.Event, + device_index: usize, + /// Bytes this submission holds against the device's in-flight budget. + len: usize, + /// Awake-clock nanoseconds of the submission. + submitted_ns: u64 = 0, + /// Link of the device pump's `retired`; owned by that pump's mutex. + next_retired: ?*EventContext = null, + + /// Pump mutex, unless no callback was registered. Destroys the + /// event, once. The platform is reached through the request, and + /// only when there is something to destroy: a context whose event + /// is null never touches it. + fn destroyEvent(self: *EventContext) void { + if (self.pjrt_event) |event| event.deinit(self.request.loader.platform.pjrt_api); + self.pjrt_event = null; + } + }; + + fn lockAllPumps(loader: *Loader) void { + for (loader.pumps) |*device_pump| device_pump.mutex.lockUncancelable(loader.io); + } + + fn unlockAllPumps(loader: *Loader) void { + var index = loader.pumps.len; + while (index > 0) { + index -= 1; + loader.pumps[index].mutex.unlock(loader.io); + } + } + + /// Retires the contexts of a done batch from the awaiting task, after + /// their callbacks fired, exactly as `pjrt.Event.await` does: drains + /// every pump's `retired`, whichever batch its contexts belong to, and + /// checks that every request completed. Runs with every pump locked so + /// an `abortReady` still iterating queued entries or a pump draining its + /// `retired` cannot race the free that follows. + fn retireBatch(loader: *Loader, batch: *Batch) void { + std.debug.assert(batch.done.isSet()); + lockAllPumps(loader); + defer unlockAllPumps(loader); + for (loader.pumps) |*device_pump| device_pump.destroyRetired(); + for (batch.plans.items) |plan| { + for (plan.requests) |*request| std.debug.assert(request.pending.load(.acquire) == 0); + } + } + + /// Hands a context whose callback fired to its device's next pump. Under + /// that pump's mutex, so the batch's retirement sees it before the batch + /// is freed. + fn retireEvent(loader: *Loader, ctx: *EventContext) void { + const device_pump = &loader.pumps[ctx.device_index]; + device_pump.mutex.lockUncancelable(loader.io); + ctx.next_retired = device_pump.retired; + device_pump.retired = ctx; + device_pump.mutex.unlock(loader.io); + } + + /// Queues every piece of the request's job before pumping: queuing piece + /// by piece costs roughly 69k-79k mutex and pump trips per DeepSeek load, + /// which coalesced reads alone do not remove. A transfer reads `len` + /// bytes from its `landing` of `blocks` taken end to end, inside one + /// block, once for each of its writers: the devices the planner planned + /// for (both come from the sharding's canonical device order). + fn enqueue(request: *ReadRequest, blocks: []const host_memory.Block) !void { + const loader = request.loader; + const block_size = loader.plan_config.block_size; + const transfers = request.plan.transfersOf(request.plan.jobs[request.index]); + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var pieces: usize = 0; + for (transfers) |transfer| { + const tensor = transfer.destination.state.readyValue().?; + pieces += @popCount(transfer.writer_mask); + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + counts[tensor.targets[writer_index].device_index] += 1; + } + } + const enqueued_ns = awakeNs(loader.io); + // Every pump is held while the request's pieces land, and every + // queue is reserved before any changes, so the request is either + // fully queued or not queued at all. + lockAllPumps(loader); + { + errdefer unlockAllPumps(loader); + // A failure recorded before these locks may have drained the + // queues already (`abortReady`), and nothing would complete what + // this queued; one recorded after them drains it. + if (loader.first_error.get()) |err| return err; + for (loader.pumps, counts[0..loader.pumps.len]) |*device_pump, count| { + try device_pump.queue.ensureUnusedCapacity(loader.allocator, count); + } + } + // Every piece holds the request, and with it the slot, until its + // callback. + _ = request.pending.fetchAdd(pieces, .acq_rel); + request.enqueued_ns = enqueued_ns; + for (transfers) |transfer| { + // The planner cut the transfers to the job's blocks. + const data = blockPiece(blocks, block_size, transfer.landing, transfer.len); + std.debug.assert(data.len == transfer.len); + const tensor = transfer.destination.state.readyValue().?; + var mask = transfer.writer_mask; + while (mask != 0) { + const writer_index: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + const target = &tensor.targets[writer_index]; + loader.pumps[target.device_index].queue.pushBackAssumeCapacity(.{ + .target = target, + .request = request, + .data = data, + .destination_offset = transfer.destination_offset, + }); + } + } + unlockAllPumps(loader); + for (counts[0..loader.pumps.len], 0..) |count, device_index| { + if (count != 0) requestPump(loader, device_index); + } + } + + /// Runs the device's pump on this thread unless one is already running. + fn requestPump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + if (device_pump.pumping or loader.first_error.get() != null) { + device_pump.mutex.unlock(loader.io); + return; + } + device_pump.pumping = true; + device_pump.mutex.unlock(loader.io); + pump(loader, device_index); + } + + /// Submits from the device's queue, in arrival order, until the device + /// has no room or nothing is queued. Any queued piece can go: nothing + /// waits for another piece. Exactly one pump runs per device at a time, + /// see `DevicePump.pumping`. + fn pump(loader: *Loader, device_index: usize) void { + const device_pump = &loader.pumps[device_index]; + while (true) { + var selected: ?Piece = null; + device_pump.mutex.lockUncancelable(loader.io); + device_pump.destroyRetired(); + if (loader.first_error.get() == null) { + if (!device_pump.hasRoom(loader.dma_budget_bytes)) { + _ = loader.metrics.pump_stops_full.fetchAdd(1, .monotonic); + } else if (device_pump.queue.popFront()) |piece| { + device_pump.active_bytes += piece.data.len; + device_pump.active_pieces += 1; + selected = piece; + } else { + _ = loader.metrics.pump_stops_empty.fetchAdd(1, .monotonic); + } + } + if (selected == null) device_pump.pumping = false; + device_pump.mutex.unlock(loader.io); + if (selected) |piece| submitOne(loader, piece) else return; + } + } + + fn submitOne(loader: *Loader, piece: Piece) void { + const device_index = piece.target.device_index; + const len = piece.data.len; + submitTransfer(loader, piece) catch |err| { + loader.recordError(err, "submit DMA transfer: device={d}, destination_offset={d}, bytes={d}", .{ device_index, piece.destination_offset, len }); + finishPiece(loader, device_index, len, null, piece.request); + }; + } + + fn submitTransfer(loader: *Loader, piece: Piece) !void { + const api = loader.platform.pjrt_api; + const target = piece.target; + const is_last = target.nextIsLast(piece.data.len); + const submit_started = awakeNs(loader.io); + const event = try target.manager.transferData( + api, + 0, + piece.data, + @intCast(piece.destination_offset), + is_last, + ); + target.noteSubmitted(piece.data.len); + + // The plan holds one event slot per planned submission; the batch + // owns it. + const plan = piece.request.plan; + const event_index = plan.events_used.fetchAdd(1, .monotonic); + std.debug.assert(event_index < plan.events.len); + const ctx = &plan.events[event_index]; + ctx.* = .{ + .request = piece.request, + .pjrt_event = event, + .device_index = target.device_index, + .len = piece.data.len, + .submitted_ns = submit_started, + }; + + _ = loader.metrics.dma_submissions.fetchAdd(1, .monotonic); + event.onReady(api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + const request = ctx_.request; + const loader_ = request.loader; + const api_ = loader_.platform.pjrt_api; + const device_index = ctx_.device_index; + const len = ctx_.len; + _ = loader_.metrics.dma_piece_ns.fetchAdd(awakeNs(loader_.io) -| ctx_.submitted_ns, .monotonic); + // The shipped plugins resolve this event without an error + // whatever happened to the copy (the C API wrapper sets the + // promise with an OK status); kept for a plugin that reports. + // The callback owns the error. + if (err) |pjrt_error| { + loader_.recordError(pjrt_error.getCode(api_).toApiError(), "complete DMA transfer: device={d}, bytes={d}, message={s}", .{ device_index, len, pjrt_error.getMessage(api_) }); + pjrt_error.deinit(api_); + } + finishPiece(loader_, device_index, len, ctx_, request); + } + }.call, ctx) catch |err| { + event.awaitRaw(api) catch {}; + ctx.destroyEvent(); + return err; + }; + _ = loader.metrics.dma_submit_ns.fetchAdd(awakeNs(loader.io) -| submit_started, .monotonic); + } + + /// Ends a submitted piece: frees its room, which may pump on this thread; + /// then retires a fired context, after that pump, so its event is never + /// destroyed inside its own callback; then releases the request last, + /// since that release may complete and free the batch. + fn finishPiece(loader: *Loader, device_index: usize, len: usize, fired: ?*EventContext, request: *ReadRequest) void { + eventCompleted(loader, device_index, len); + if (fired) |ctx| retireEvent(loader, ctx); + request.release(); + } + + fn eventCompleted(loader: *Loader, device_index: usize, len: usize) void { + const device_pump = &loader.pumps[device_index]; + device_pump.mutex.lockUncancelable(loader.io); + std.debug.assert(device_pump.active_bytes >= len); + std.debug.assert(device_pump.active_pieces > 0); + device_pump.active_bytes -= len; + device_pump.active_pieces -= 1; + device_pump.mutex.unlock(loader.io); + // A ready callback can be the first place an asynchronous PJRT error + // becomes visible. Once outside the pump lock, retire every queued + // piece so request lifecycles cannot wait forever on entries that + // the failed pumps will no longer submit. + if (loader.first_error.get() != null) + abortReady(loader) + else + requestPump(loader, device_index); + } + + /// Force-completes every queued piece, one device at a time. A + /// completion here may finish its batch, but a batch still has an entry + /// queued until its last one is popped, and it is retired with every + /// pump locked, so no queued entry is touched after its request ends. + fn abortReady(loader: *Loader) void { + for (loader.pumps) |*device_pump| { + device_pump.mutex.lockUncancelable(loader.io); + while (device_pump.queue.popFront()) |piece| { + piece.request.release(); + } + device_pump.mutex.unlock(loader.io); + } + } +}; + +const DestinationState = struct { + /// One destination buffer of a tensor. PJRT makes the buffer ready once + /// every transfer submitted to it has completed and one of them carried + /// the last-transfer flag, whatever their completion order: the flag + /// only closes the buffer to further calls. The pump, the only + /// submitter, therefore flags the submission that completes the + /// placement's bytes, and no piece ever waits for another one. + /// Flagging the highest-offset piece instead makes a tail wait for its + /// prefixes, which can deadlock a sharded multi-device load: tails can + /// hold every lifecycle credit while their prefixes stay unclaimed, + /// whatever the pinned memory. This follows XLA's + /// CommonAsyncHostToDeviceTransferManager: readiness requires zero + /// in-flight transfers and a last-call flag, not destination-offset order. + const Target = struct { + manager: *pjrt.AsyncHostToDeviceTransferManager, + device_index: usize, + /// Bytes of the placement on this device. The pieces partition them: + /// those of every part, from whichever file and plan, and of the + /// zero fill. + total: usize, + /// Owned by the pump; read by the awaiting task once the batch is + /// done and nothing submits any more. + submitted_bytes: usize = 0, + + /// Whether a submission of `len` bytes closes the buffer. + fn nextIsLast(self: *const Target, len: usize) bool { + std.debug.assert(len != 0 and self.submitted_bytes + len <= self.total); + return self.submitted_bytes + len == self.total; + } + + fn noteSubmitted(self: *Target, len: usize) void { + std.debug.assert(self.submitted_bytes + len <= self.total); + self.submitted_bytes += len; + } + + /// Every byte went out, so the last-transfer flag did too. + fn fullySubmitted(self: *const Target) bool { + return self.submitted_bytes == self.total; + } + }; + + /// A DMA destination's transfer targets, one per device in canonical + /// order; empty for a host-resident one. + targets: []Target = &.{}, + /// A host-resident destination's shards in host memory, which workers + /// write in place, one per device in canonical order; empty for a DMA + /// one. The devices of one box share theirs (`initHost`). + host: [][*]u8 = &.{}, + + /// Creates the destination's buffers and writes the output shell: for a + /// DMA destination, the device buffers and transfer managers, one per + /// device of its grid in canonical order; for a host-resident one, the + /// buffers workers write (`initHost`). Runs once per destination + /// (`LazyOnce`), however many parts feed it, on the allocation task or + /// the worker that first reads or fills for it, which also counts the + /// allocation once. A host destination (`Memory.isHost`) is created in + /// each device's pinned host memory space and takes no device memory, + /// so it is not counted in `allocated_bytes`. + fn init(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + if (destination.memory.isHost()) return initHost(direct, destination); + const allocator = direct.allocator; + const platform = direct.platform; + const grid = &destination.grid; + const targets = try allocator.alloc(Target, grid.devices.len); + errdefer allocator.free(targets); + + var pjrt_buffers: Buffer.Shards = .empty; + var initialized: usize = 0; + errdefer { + for (targets[0..initialized]) |target| target.manager.deinit(platform.pjrt_api); + for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(platform.pjrt_api); + } + + const shape_spec: pjrt.ShapeSpec = .init( + grid.local.dims(), + pjrtx.bufferTypeFromDtype(grid.local.dtype()), + ); + for (grid.devices, 0..) |device, i| { + const memory = platform.devices[device.id].memory(.default).?; + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, + }); + errdefer manager.deinit(platform.pjrt_api); + const pjrt_buffer = try manager.retrieveBuffer(platform.pjrt_api, 0); + targets[i] = .{ + .manager = manager, + .device_index = device.id, + .total = grid.local.byteSize(), + }; + initialized += 1; + pjrt_buffers.appendAssumeCapacity(pjrt_buffer); + } + for (targets) |target| { + _ = direct.allocated_bytes[target.device_index].fetchAdd(target.total, .monotonic); + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .targets = targets }; + } + + /// Creates a host-resident destination's buffers, one per device of its + /// grid in canonical order, each in that device's pinned host memory + /// space, and reads their host pointers for the workers. The devices + /// holding one box (`ShardGrid.writers`) share one allocation: every + /// device's shard is a view of it, and the allocation lives until the + /// last view is gone (`SharedHostAllocation`). A transfer is then + /// written once, through its first writer's shard (`hostLanding`). A + /// `host_mapped` shard is always such a view, on the device's default + /// memory space, so kernels read the allocation in place. Every direct + /// target lets a host allocation of one device back a view on another, + /// for transfers and kernels alike: CPU, CUDA and ROCm allocate portable + /// pinned memory (`Buffer.HostAccessible.init`), and on oneAPI four B70 + /// gather from an allocation made in the first one's host memory space, + /// and read it back. A target that could not would need a copy per + /// device. + /// The buffers are complete before any worker writes them, so they + /// report ready from the start; the front end hands them out only once + /// the batch is done. + fn initHost(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { + const allocator = direct.allocator; + const platform = direct.platform; + const api = platform.pjrt_api; + const grid = &destination.grid; + const shard_count = grid.devices.len; + std.debug.assert(shard_count <= Platform.MAX_NUM_DEVICES); + const shards = try allocator.alloc([*]u8, shard_count); + errdefer allocator.free(shards); + const dims = grid.local.dims(); + const element_type = pjrtx.bufferTypeFromDtype(grid.local.dtype()); + const layout = platform.defaultMemoryLayout(dims, grid.local.dtype()); + + // The writers of each device's box; the first of them creates the + // allocation the others view. + var box_writers: [Platform.MAX_NUM_DEVICES]u64 = undefined; + for (grid.writers[0..grid.box_count]) |writers| { + var mask = writers; + while (mask != 0) : (mask &= mask - 1) box_writers[@ctz(mask)] = writers; + } + + // Creation holds a reference to every shared allocation and its + // views one each. Declared before the buffers' cleanup, so a failure + // destroys the views first and this frees the owners. + var shared: [Platform.MAX_NUM_DEVICES]?*SharedHostAllocation = @splat(null); + defer for (shared[0..shard_count]) |allocation| if (allocation) |a| a.release(); + var pjrt_buffers: Buffer.Shards = .empty; + errdefer for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(api); + for (grid.devices, shards, 0..) |device, *shard, index| { + // The front end only submits host kinds when every device has + // a pinned host space (`backend.supportsMemory`). + const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + if (grid.replicas() > 1 or destination.memory == .host_mapped) { + const first: usize = @ctz(box_writers[index]); + if (first == index) shared[index] = try .create(direct.io, platform, dims, element_type, layout, host); + const allocation = shared[first].?; + const space = platform.devices[device.id].memory(destination.memory.space()).?.pjrt_memory; + pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, space)); + shard.* = allocation.data; + } else { + const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); + pjrt_buffers.appendAssumeCapacity(buffer); + shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); + } + } + + destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); + return .{ .host = shards }; + } + + fn deinit(self: *DestinationState, allocator: std.mem.Allocator, api: *const pjrt.Api) void { + for (self.targets) |target| target.manager.deinit(api); + allocator.free(self.targets); + allocator.free(self.host); + } + + /// Where a host transfer's bytes go: the allocation its writers, the + /// devices of one box, share. Writing it once writes every one of them. + fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer) []u8 { + const first: usize = @ctz(transfer.writer_mask); + std.debug.assert(first < self.host.len); + return self.host[first][transfer.destination_offset..][0..transfer.len]; + } +}; + +/// One host allocation that the devices of a box read through views of it, +/// in their pinned host memory space or, for `host_mapped`, their default +/// one: the owner buffer is not a shard, every view holds a reference that +/// its deletion callback drops, and the last drop destroys the owner. PJRT +/// runs a view's callback once the view's memory is out of use, after any +/// execution still reading it, so the allocation outlives its readers +/// whatever order the shards are freed in, including views of the shards +/// (`Buffer.createView`) that their owner outlives. On the process heap: +/// the last view may go long after the loader. +/// Writing through one view shows on every device, so a shard of a +/// replicated host-resident destination must not be donated to an +/// execution that writes it, as with `Buffer.HostAccessible.init(.undonatable)`. +const SharedHostAllocation = struct { + /// Allocations not released yet, counted in tests only. + var live: if (builtin.is_test) std.atomic.Value(usize) else void = if (builtin.is_test) .init(0) else {}; + + api: *const pjrt.Api, + owner: *pjrt.Buffer, + data: [*]u8, + /// One per live view, and one for `DestinationState.initHost` while it + /// creates them. + references: std.atomic.Value(usize) = .init(1), + + fn create( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*SharedHostAllocation { + const api = platform.pjrt_api; + const self = try std.heap.smp_allocator.create(SharedHostAllocation); + errdefer std.heap.smp_allocator.destroy(self); + const owner = try createHostBuffer(io, platform, dims, element_type, layout, memory); + errdefer owner.deinit(api); + self.* = .{ + .api = api, + .owner = owner, + .data = @ptrCast(try owner.opaqueDeviceMemoryDataPointer(api)), + }; + if (builtin.is_test) _ = live.fetchAdd(1, .monotonic); + return self; + } + + /// A shard in `memory`, holding a reference: the host space of its + /// device, or the default one, where kernels read the allocation in + /// place. Every device can read it. + fn view( + self: *SharedHostAllocation, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, + ) StateInitError!*pjrt.Buffer { + _ = self.references.fetchAdd(1, .monotonic); + errdefer self.release(); + return platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = self.data, + .dims = dims, + .element_type = element_type, + .layout = layout, + .memory = memory, + .on_delete_callback = &onViewDeleted, + .on_delete_callback_arg = self, + }); + } + + fn onViewDeleted(_: ?*anyopaque, context: ?*anyopaque) callconv(.c) void { + const self: *SharedHostAllocation = @ptrCast(@alignCast(context.?)); + self.release(); + } + + fn release(self: *SharedHostAllocation) void { + if (self.references.fetchSub(1, .acq_rel) != 1) return; + self.owner.deinit(self.api); + std.heap.smp_allocator.destroy(self); + if (builtin.is_test) _ = live.fetchSub(1, .monotonic); + } +}; + +/// Shared host allocations not released yet, in tests: the front end's +/// tests count the ones their views keep. +pub fn liveHostAllocations() usize { + comptime std.debug.assert(builtin.is_test); + return SharedHostAllocation.live.load(.acquire); +} + +/// An uninitialized buffer in a host memory space, once its allocation +/// completed: its host pointer is valid only then. +fn createHostBuffer( + io: std.Io, + platform: *const Platform, + dims: []const i64, + element_type: pjrt.BufferType, + layout: pjrt.MemoryLayout, + memory: *const pjrt.Memory, +) StateInitError!*pjrt.Buffer { + const api = platform.pjrt_api; + const buffer = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = dims, + .element_type = element_type, + .layout = layout, + .dst = .{ .memory = memory }, + }); + errdefer buffer.deinit(api); + const ready = buffer.readyEvent(api); + defer ready.deinit(api); + try ready.await(api, io); + return buffer; +} + +/// The lifecycle credits: one per request the pre-grown pinned set holds, +/// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit +/// before claiming a job, and the request gives it back with its last +/// reference: after its last DMA callback, or at the end of a host job. Only +/// the holder reads into, lands in and transfers from its slot's blocks, so +/// the credits are the one bound on pinned memory in use. A block whose +/// pieces are done stays with its request until the request ends; nobody +/// else could use it anyway, since every credit owns a whole slot. +const Credits = struct { + /// Every slot's blocks, slot `s` at `blocks[s * blocks_per_slot ..]`; + /// a tail shorter than a slot is left unused. + blocks: []const host_memory.Block, + blocks_per_slot: usize, + /// Free slots, a stack of indices; owned by `mutex`. + free: []u32, + free_len: usize, + /// The most slots held at once. + high_water: usize = 0, + mutex: std.Io.Mutex = .init, + /// Waiters for a slot; one release wakes one of them. + condition: std.Io.Condition = .init, + + fn init(allocator: std.mem.Allocator, blocks: []const host_memory.Block, blocks_per_slot: usize) std.mem.Allocator.Error!Credits { + std.debug.assert(blocks_per_slot != 0); + const free = try allocator.alloc(u32, blocks.len / blocks_per_slot); + // Slot 0 first. + for (free, 0..) |*slot, index| slot.* = @intCast(free.len - 1 - index); + return .{ .blocks = blocks, .blocks_per_slot = blocks_per_slot, .free = free, .free_len = free.len }; + } + + fn deinit(self: *Credits, allocator: std.mem.Allocator) void { + std.debug.assert(self.free_len == self.free.len); + allocator.free(self.free); + } + + fn count(self: *const Credits) usize { + return self.free.len; + } + + /// Waits for a free slot. Requests in flight give theirs back even after + /// a failure, so a waiter always gets one. + fn acquire(self: *Credits, io: std.Io) u32 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.free_len == 0) { + self.condition.waitUncancelable(io, &self.mutex); + } + self.free_len -= 1; + self.high_water = @max(self.high_water, self.free.len - self.free_len); + return self.free[self.free_len]; + } + + fn release(self: *Credits, io: std.Io, slot: u32) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + std.debug.assert(self.free_len < self.free.len); + self.free[self.free_len] = slot; + self.free_len += 1; + // One release frees one slot: waking every waiter here would be a + // thundering herd. + self.condition.signal(io); + } + + fn slotBlocks(self: *const Credits, slot: u32) []const host_memory.Block { + return self.blocks[slot * self.blocks_per_slot ..][0..self.blocks_per_slot]; + } + + fn inUse(self: *Credits, io: std.Io) usize { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + return self.free.len - self.free_len; + } +}; + +/// The debug summary's counters (`Loader.logSummary`): read only by logs; +/// nothing here decides. +const Metrics = struct { + read_operations: std.atomic.Value(u64) = .init(0), + read_bytes: std.atomic.Value(u64) = .init(0), + dma_submissions: std.atomic.Value(u64) = .init(0), + /// Time workers spend waiting for a lifecycle credit and inside the + /// source read, summed over requests: waits above the read time mean the + /// load is DMA-completion bound. + credit_wait_ns: std.atomic.Value(u64) = .init(0), + read_ns: std.atomic.Value(u64) = .init(0), + /// From a request's enqueue to its last DMA callback: how long the DMA + /// stage holds a lifecycle credit and pinned blocks per request. + dma_stage_ns: std.atomic.Value(u64) = .init(0), + /// Inside `submitTransfer` (the PJRT call and the callback registration) + /// and from a submission to its ready callback, summed over submissions. + dma_submit_ns: std.atomic.Value(u64) = .init(0), + dma_piece_ns: std.atomic.Value(u64) = .init(0), + /// Why a device pump stopped: nothing queued, or no room under the + /// device's budget. A pump that mostly stops empty is starved by the + /// reads; one that mostly stops full is waiting on the engine. + pump_stops_empty: std.atomic.Value(u64) = .init(0), + pump_stops_full: std.atomic.Value(u64) = .init(0), + /// Inside `ensureState` for a claimed job's destinations: the first + /// worker to touch a tensor creates its PJRT buffers and transfer + /// managers there, and the other workers of the same tensor wait for it; + /// a host job waits there for the allocation task. + tensor_init_ns: std.atomic.Value(u64) = .init(0), + /// The allocation task's time creating host-resident destinations ahead + /// of their reads, summed over batches. + ahead_init_ns: std.atomic.Value(u64) = .init(0), +}; + +/// A value initialized at most once by whichever task touches it first, +/// with the initializer that task passes. Concurrent callers wait on the +/// event; a failed initialization keeps its error code and re-materializes +/// the same error for every later caller. +fn LazyOnce(comptime T: type) type { + return struct { + const Self = @This(); + const Status = enum(u8) { + uninitialized, + initializing, + ready, + failed, + }; + + value: T = undefined, + status: std.atomic.Value(Status) = .init(.uninitialized), + error_code: std.atomic.Value(u16) = .init(0), + initialized: std.Io.Event = .unset, + + /// The error set of `initFn`, which returns `T`. + fn InitError(comptime initFn: anytype) type { + return @typeInfo(@typeInfo(@TypeOf(initFn)).@"fn".return_type.?).error_union.error_set; + } + + fn ensure(self: *Self, io: std.Io, comptime initFn: anytype, args: std.meta.ArgsTuple(@TypeOf(initFn))) InitError(initFn)!*T { + while (true) switch (self.status.load(.acquire)) { + .uninitialized => { + if (self.status.cmpxchgStrong(.uninitialized, .initializing, .acq_rel, .acquire) != null) continue; + self.value = @call(.auto, initFn, args) catch |err| { + self.error_code.store(@intFromError(err), .release); + self.status.store(.failed, .release); + self.initialized.set(io); + return err; + }; + self.status.store(.ready, .release); + self.initialized.set(io); + return &self.value; + }, + .initializing => self.initialized.waitUncancelable(io), + .ready => return &self.value, + .failed => return @errorCast(@errorFromInt(self.error_code.load(.acquire))), + }; + } + + /// The value when initialization has completed successfully. + fn readyValue(self: *Self) ?*T { + return if (self.status.load(.acquire) == .ready) &self.value else null; + } + }; +} + +fn secondsBetween(from: std.Io.Timestamp, to: std.Io.Timestamp) f64 { + return @as(f64, @floatFromInt(from.durationTo(to).nanoseconds)) / std.time.ns_per_s; +} + +/// Mean milliseconds per unit of a summed duration (zero units: the total). +fn millisecondsPer(total_ns: u64, units: u64) f64 { + return @as(f64, @floatFromInt(total_ns)) / std.time.ns_per_ms / @as(f64, @floatFromInt(@max(units, 1))); +} + +fn awakeNs(io: std.Io) u64 { + const now: std.Io.Timestamp = .now(io, .awake); + return @intCast(@max(now.nanoseconds, 1)); +} + +/// Writes `contents` to a new file `name` in `dir` and returns its real +/// path, in `path_buffer`. +fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []const u8, path_buffer: []u8) ![]const u8 { + const file = try dir.createFile(io, name, .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, contents, 0); + return path_buffer[0..try file.realPath(io, path_buffer)]; +} + +// Planner tests. + +/// Every file read at `alignment`, none opened to ask. +fn fixedReads(alignment: usize) Planner.Reads { + return .{ .io = undefined, .vfs = null, .policy = .off, .alignment = alignment }; +} + +/// What a loader's `Planner.Reads` predicts for `checkpoint` under `policy`. +fn testPrediction(io: std.Io, policy: VFS.DirectIo, checkpoint: ?[]const u8) !Planner.Reads.Prediction { + var reads: Planner.Reads = .init(io, policy); + defer reads.deinit(std.testing.allocator); + return reads.predict(std.testing.allocator, checkpoint); +} + +/// `Planner.preparePlan` with every file read at `alignment`, kept in +/// `arena`; the testing allocator checks the cut for leaks. +fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config, alignment: usize) !*Plan { + return Planner.preparePlan(std.testing.allocator, arena, parts, order, config, fixedReads(alignment)); +} + +/// One plain destination per source, as `createDestinations` builds them +/// for whole-tensor specs, without device state: for planner tests. +fn initPlainParts( + platform: *const Platform, + sources: []safetensors.Tensor, + slots: []const *SourceSlot, + outputs: []Buffer, + destinations: []Destination, + parts: []Part, +) void { + for (sources, slots, outputs, destinations, parts) |*source, slot, *output, *destination, *part| { + part.* = .{ + .source = source, + .source_slot = slot, + .destination = destination, + .lattice = .{ .width = source.byteSize() }, + }; + destination.* = .{ + .shape = source.shape, + .grid = .init(platform, source.shape), + .output = output, + .parts = @as(*[1]Part, part), + .unread_bytes = .init(source.byteSize()), + }; + } +} + +/// Claims out and frees an open batch that a test only inspected; it must +/// be the only queued batch. +fn discardTestBatch(scheduler: *Scheduler, batch: *Batch) void { + const io = std.testing.io; + var claimed: usize = 0; + while (scheduler.claim(io)) |claim| { + std.debug.assert(claim.plan.batch == batch); + claimed += 1; + } + batch.releaseUnits(1 + claimed); + std.debug.assert(batch.done.isSet()); + batch.destroy(); +} + +test "the planner opens only local files to ask about direct I/O" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents: [2 * 4096]u8 = @splat(7); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "drafter.bin", &contents, &path_buffer); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A scheme whose files the plain io cannot open: asking would fail. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + const reads: Planner.Reads = .{ .io = vfs_io, .vfs = &vfs, .policy = .on, .alignment = 4096 }; + try std.testing.expectEqual(@as(usize, 0), reads.boundOf("remote://bucket/model.safetensors")); + try std.testing.expectEqual(@as(usize, 4096), reads.boundOf(path)); + + // A remote file is read exact, without an open on the planner. + var remote: SourceSlot = .{ .uri = "remote://bucket/model.safetensors" }; + defer remote.deinit(vfs_io); + try std.testing.expectEqual(@as(usize, 0), try reads.alignmentOf(&remote)); + try std.testing.expect(remote.file.readyValue() == null); + + // A local file beside it is opened and asked, whatever the checkpoint: + // direct where the filesystem allows it. + var local: SourceSlot = .{ .uri = path }; + defer local.deinit(vfs_io); + const alignment = try reads.alignmentOf(&local); + const file = local.file.readyValue().?.*; + try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); +} + +test "the checkpoint's reads are predicted as the VFS will decide the files its registry reads" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const long: [2 * VFS.direct_io_alignment + 100]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "a.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "b.safetensors", .data = &long }); + try tmp.dir.writeFile(io, .{ .sub_path = "c.safetensors", .data = long[0..100] }); + // The index lists a, b, c and a symlink to a; the registry does not + // read the consolidated file beside them (Mistral's format). The files + // hold no header: none is read. + try tmp.dir.symLink(io, "a.safetensors", "alias.safetensors", .{}); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"x\":\"a.safetensors\",\"y\":\"b.safetensors\",\"z\":\"c.safetensors\",\"w\":\"alias.safetensors\"}}" }); + try tmp.dir.writeFile(io, .{ .sub_path = "consolidated.safetensors", .data = &long }); + // A snapshot whose file links into blobs, as a Hugging Face cache's do. + const blob: [8 * 1024]u8 = @splat(9); + try tmp.dir.createDir(io, "blobs", .default_dir); + try tmp.dir.writeFile(io, .{ .sub_path = "blobs/x", .data = &blob }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 64]u8 = undefined; + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // A remote scheme: nothing is predicted for a checkpoint there. + try vfs.registerBackend("remote", .{ .io = io }); + const vfs_io = vfs.io(); + + // The listed files as fresh handles decide them under `on`: c, shorter + // than a unit, never direct. + var expected: Planner.Reads.Prediction = .{}; + const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); + defer dir.close(vfs_io); + for ([_][]const u8{ "a.safetensors", "b.safetensors", "c.safetensors" }) |name| { + const file = try dir.openFile(vfs_io, name, .{}); + defer file.close(vfs_io); + const size = (try file.stat(vfs_io)).size; + expected.files += 1; + expected.bytes += size; + if (vfs.useDirectIo(file, .on)) expected.direct_bytes += size; + } + try std.testing.expectEqual(@as(u64, 2 * long.len + 100), expected.bytes); + + var on_reads: Planner.Reads = .init(vfs_io, .on); + defer on_reads.deinit(allocator); + // The symlink is predicted once, with a. + try std.testing.expectEqual(expected, try on_reads.predict(allocator, root)); + // Each answer is kept by the file's real path, none for the + // consolidated file. Under `on` each keeps `on`, c too: planning + // decides it, and it reads buffered. + try std.testing.expectEqual(@as(usize, 3), on_reads.predicted.count()); + try std.testing.expectEqual(VFS.DirectIo.on, on_reads.predicted.get(try std.fmt.bufPrint(&path_buffer, "{s}/c.safetensors", .{root})).?); + try std.testing.expectEqual(expected, try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors.index.json", .{root}))); + const one = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/a.safetensors", .{root})); + try std.testing.expectEqual(@as(usize, 1), one.files); + try std.testing.expectEqual(@as(u64, long.len), one.bytes); + const snapshot = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root})); + try std.testing.expectEqual(@as(usize, 1), snapshot.files); + try std.testing.expectEqual(@as(u64, blob.len), snapshot.bytes); + + // Just written, so cached: none reads direct under `auto`, and each is + // kept as buffered. + var auto_reads: Planner.Reads = .init(vfs_io, .auto); + defer auto_reads.deinit(allocator); + const cached = try auto_reads.predict(allocator, root); + try std.testing.expectEqual(@as(usize, 3), cached.files); + try std.testing.expectEqual(@as(u64, 0), cached.direct_bytes); + var answers = auto_reads.predicted.valueIterator(); + while (answers.next()) |answer| try std.testing.expectEqual(VFS.DirectIo.off, answer.*); + + // Nothing to predict: no direct I/O, no VFS, a remote or no checkpoint, + // or none there. + const nothing = [_]Planner.Reads.Prediction{ + try testPrediction(vfs_io, .off, root), + try testPrediction(io, .on, root), + try testPrediction(vfs_io, .on, "remote://bucket/model"), + try testPrediction(vfs_io, .on, null), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/missing", .{root})), + try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/blobs", .{root})), + }; + for (nothing) |prediction| try std.testing.expectEqual(@as(usize, 0), prediction.files); +} + +test "a file predicted to read direct is planned direct, though the prediction's sample cached it" { + if (comptime VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // A snapshot whose file links into blobs, as a Hugging Face cache's + // does. 64 KiB: every page of it is sampled, on 4 KiB pages too. + const data: [64 * 1024]u8 = @splat(7); + const blobs = try tmp.dir.createDirPathOpen(io, "blobs", .{}); + defer blobs.close(io); + try testing.writeSafetensorsFile(allocator, io, blobs, "x", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{data.len}, .bytes = &data }, + }); + try tmp.dir.createDir(io, "snapshot", .default_dir); + try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; + const checkpoint = try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root}); + + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const vfs_io = vfs.io(); + // The planner's slot takes the registry's name for the file: the blob. + var registry: safetensors.TensorRegistry = try .fromPath(allocator, vfs_io, checkpoint); + defer registry.deinit(); + const uri = registry.tensors.get("w").?.file_uri; + { + const file = try blobs.openFile(io, "x", .{}); + defer file.close(io); + try file.sync(io); + _ = std.os.linux.fadvise(file.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); + } + + var reads: Planner.Reads = .init(vfs_io, .auto); + defer reads.deinit(allocator); + const predicted = try reads.predict(allocator, checkpoint); + // A filesystem that kept the file cached (tmpfs) or refuses direct + // reads says nothing here. + if (predicted.direct_bytes == 0) return error.SkipZigTest; + // Once the sample's reads landed, a probe of its own finds the file + // cached (a kernel that queues no read for a missed sample shows + // nothing). + try io.sleep(.fromMilliseconds(250), .awake); + { + const file = try std.Io.Dir.openFile(.cwd(), vfs_io, uri, .{ .mode = .read_only }); + defer file.close(vfs_io); + if (vfs.useDirectIo(file, .auto)) return error.SkipZigTest; + } + + // Planning reads it as predicted: direct, the handle decided so. + var slot: SourceSlot = .{ .uri = uri }; + defer slot.deinit(vfs_io); + try std.testing.expectEqual(reads.alignment, try reads.alignmentOf(&slot)); + try std.testing.expect(vfs.useDirectIo(slot.file.readyValue().?.*, .auto)); +} + +test "arenas are placed for direct reads from four fifths of the checkpoint's bytes" { + const Case = struct { direct_bytes: u64, bytes: u64, direct: bool }; + const cases = [_]Case{ + .{ .direct_bytes = 0, .bytes = 0, .direct = false }, + .{ .direct_bytes = 4, .bytes = 5, .direct = true }, + .{ .direct_bytes = 79, .bytes = 100, .direct = false }, + .{ .direct_bytes = 80, .bytes = 100, .direct = true }, + .{ .direct_bytes = 0, .bytes = 100, .direct = false }, + }; + for (cases) |case| { + const prediction: Planner.Reads.Prediction = .{ .bytes = case.bytes, .direct_bytes = case.direct_bytes }; + try std.testing.expectEqual(case.direct, prediction.direct()); + } +} + +test "a strided part whose source runs are not back to back counts its whole source range" { + // [32, 16] u8 at a file offset off a 64-byte alignment. + const source: safetensors.Tensor = .{ .file_uri = "wide.bin", .name = "wide", .shape = .init(.{ 32, 16 }, .u8), .offset = 100 }; + const config: Planner.Config = .{ .block_size = 256, .request_size = 512 }; + // Transposed, 512 runs of one byte: every job reads all 512 bytes, + // 576 widened to the alignment. + const transposed: LoadPart = .{ .source = &source, .lattice = .{ .width = 1, .dims = &.{ + .{ .count = 16, .src_stride = 1, .dst_stride = 32 }, + .{ .count = 32, .src_stride = 16, .dst_stride = 1 }, + } } }; + try std.testing.expectEqual(@as(backend.Backend.StridedRead, .{ .read = 512, .limit = 512 }), Planner.stridedRead(config, fixedReads(0), transposed)); + try std.testing.expectEqual(576, Planner.stridedRead(config, fixedReads(64), transposed).read); + // A bigger block calibration picked does not raise the limit past the + // smallest one's request. + const bigger_block: Planner.Config = .{ .block_size = 1024, .request_size = 1024, .gather_limit = 512 }; + try std.testing.expectEqual(512, Planner.stridedRead(bigger_block, fixedReads(0), transposed).limit); + // Every other row, in order: the gaps stretch a job's range, so the + // whole range counts, from the first row to the last. + const every_other_row: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 16, .src_stride = 32, .dst_stride = 16 }} } }; + try std.testing.expectEqual(15 * 32 + 16, Planner.stridedRead(config, fixedReads(0), every_other_row).read); + // Rows back to back read within their job's range. + const rows: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 32, .src_stride = 16, .dst_stride = 16 }} } }; + try std.testing.expectEqual(0, Planner.stridedRead(config, fixedReads(64), rows).read); +} + +test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 14 }, + .{ .file_uri = "a", .name = "a0-copy", .shape = .init(.{4}, .u8), .offset = 10 }, + .{ .file_uri = "a", .name = "a-gap", .shape = .init(.{4}, .u8), .offset = 20 }, + .{ .file_uri = "b", .name = "b0", .shape = .init(.{12}, .u8), .offset = 3 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[0], &slots[0], &slots[0], &slots[0], &slots[1] }, &outputs, &destinations, &parts); + + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 8, + }, fixedReads(0)); + + // One plan per file. a:[10,18) merges adjacency and the duplicate, + // a:[20,24) remains exact, and b:[3,15) is split at the request-size + // boundary. + const plans = batch.plans.items; + try std.testing.expectEqual(@as(usize, 2), plans.len); + try std.testing.expect(plans[0].source_slot == &slots[0]); + try std.testing.expect(plans[1].source_slot == &slots[1]); + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs.len); + try std.testing.expectEqual(@as(usize, 4), plans[0].transfers.len); + try std.testing.expectEqual(@as(u64, 10), plans[0].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[0].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 3), plans[0].jobs[0].transfer_len); + try std.testing.expectEqual(@as(u64, 20), plans[0].jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].transfers.len); + try std.testing.expectEqual(@as(u64, 3), plans[1].jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), plans[1].jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 4), plans[1].jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(u64, 12), plans[0].source_bytes); + try std.testing.expectEqual(@as(u64, 12), plans[1].source_bytes); + // One writer per transfer on one device: one event per transfer. Jobs + // span their 4-byte blocks. + try std.testing.expectEqual(@as(usize, 4), plans[0].events.len); + try std.testing.expectEqual(@as(usize, 3), plans[1].events.len); + try std.testing.expectEqual(@as(usize, 2), plans[0].requests.len); + try std.testing.expectEqual(@as(usize, 2), plans[0].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); + try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); + for (plans) |plan| { + for (plan.requests) |*request| try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + } + discardTestBatch(&scheduler, batch); + + var aligned_sources = [_]safetensors.Tensor{ + .{ .file_uri = "aligned", .name = "aligned0", .shape = .init(.{7}, .u8), .offset = 0 }, + .{ .file_uri = "aligned", .name = "aligned1", .shape = .init(.{8}, .u8), .offset = 7 }, + .{ .file_uri = "aligned", .name = "aligned2", .shape = .init(.{5}, .u8), .offset = 15 }, + }; + var aligned_slot: SourceSlot = .{ .uri = "aligned" }; + var aligned_outputs: [aligned_sources.len]Buffer = undefined; + var aligned_destinations: [aligned_sources.len]Destination = undefined; + var aligned_items: [aligned_sources.len]Part = undefined; + initPlainParts(platform, &aligned_sources, &.{ &aligned_slot, &aligned_slot, &aligned_slot }, &aligned_outputs, &aligned_destinations, &aligned_items); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 8, + }, 0); + try std.testing.expectEqual(@as(usize, 3), aligned_plan.jobs.len); + try std.testing.expectEqual(@as(usize, 6), aligned_plan.transfers.len); + try std.testing.expectEqual(@as(usize, 7), aligned_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); + + // Widened for direct I/O: the same three jobs (16 - 2 * 4 = 8 still cuts + // at the tensor-safe boundaries 7 and 15), each read at 4-byte bounds, + // its transfers relative to the widened start; the last read runs past + // the tensor data, of which every byte is required. + const widened_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ + .block_size = 4, + .request_size = 16, + }, 4); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs.len); + try std.testing.expectEqual(@as(u64, 0), widened_plan.jobs[0].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[0].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 7), widened_plan.jobs[0].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(u64, 4), widened_plan.jobs[1].file_offset); + try std.testing.expectEqual(@as(usize, 12), widened_plan.jobs[1].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 11), widened_plan.jobs[1].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs[1].blockLen(4)); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[1].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 12), widened_plan.jobs[2].file_offset); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.len); + try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.minimum_len); + try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[2].transfer_start].landing); + try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); +} + +test "host-resident and DMA parts never share a job, and overlapping host parts read through blocks" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 2 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + // x, y and z end to end in one file. x lands in two host destinations, + // y in device memory between them, z in a third host destination. + const shape = Shape.init(.{ 8, 8 }, .u8); + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "file", .name = "x", .shape = shape, .offset = 0 }, + .{ .file_uri = "file", .name = "y", .shape = shape, .offset = 64 }, + .{ .file_uri = "file", .name = "z", .shape = shape, .offset = 128 }, + }; + var slot: SourceSlot = .{ .uri = "file" }; + var outputs: [4]Buffer = undefined; + var destinations: [4]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[0], .lattice = .{ .width = 64 } }, + .{ .source = &sources[1], .source_slot = &slot, .destination = &destinations[1], .lattice = .{ .width = 64 } }, + .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[2], .lattice = .{ .width = 64 } }, + .{ .source = &sources[2], .source_slot = &slot, .destination = &destinations[3], .lattice = .{ .width = 64 } }, + }; + const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned }; + for (&destinations, &outputs, &parts, memories) |*destination, *output, *part, memory| { + destination.* = .{ .shape = shape, .grid = .init(platform, shape), .memory = memory, .output = output, .parts = @as(*[1]Part, part), .unread_bytes = .init(64) }; + } + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const plan = try prepareTestPlan(arena.allocator(), &parts, order, .{ + .block_size = 64, + .request_size = 1024, + }, 0); + // x's run reads its bytes once for two destinations, so through + // blocks; y goes out by DMA; z is read in place. File order. + const expected = [_]struct { offset: u64, kind: std.meta.Tag(Plan.Job.Kind) }{ + .{ .offset = 0, .kind = .host_bounce }, + .{ .offset = 64, .kind = .dma_read }, + .{ .offset = 128, .kind = .host_direct }, + }; + try std.testing.expectEqual(expected.len, plan.jobs.len); + var dma_writers: usize = 0; + for (plan.jobs, expected) |job, want| { + try std.testing.expectEqual(want.offset, job.file_offset); + try std.testing.expectEqual(want.kind, std.meta.activeTag(job.kind)); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + try std.testing.expectEqual(job.kind != .dma_read, transfer.destination.memory.isHost()); + if (job.kind == .dma_read) dma_writers += @popCount(transfer.writer_mask); + } + } + try std.testing.expectEqual(@as(usize, 2), plan.jobs[0].transfer_len); + try std.testing.expectEqual(@as(usize, 1), plan.jobs[0].blockLen(64)); + try std.testing.expectEqual(dma_writers, plan.events.len); +} + +/// Plans a composite destination fed by three parts from two files, with a +/// gap and a tail of zero fill, beside a plain one, then replays the plans on +/// host memory: every device's placement bytes are written exactly once, +/// from the right source bytes or from zeros. +const CompositeTest = struct { + const Options = struct { + block_size: usize, + request_size: usize, + memory: Memory = .default, + alignment: usize = 0, + /// By default nothing is staged. + staging_gain: usize = std.math.maxInt(usize), + /// Whether some piece holds a's last byte and b's first one. + cross_part_piece: bool, + }; + + /// With a host memory kind the destinations take host jobs: direct + /// without `alignment`, through its slot's blocks with it. A DMA job is + /// staged where it saves `staging_gain` submissions; the replay reads + /// its blocks from the two files as its worker would (`jobBlocks`). + fn run(platform: *const Platform, shape: Shape, options: Options) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const block_size = options.block_size; + var first_bytes: [96]u8 = undefined; + for (&first_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 31 +% 7)) | 1; + var second_bytes: [128]u8 = undefined; + for (&second_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 17 +% 3)) | 0x80; + // Rows of 16 bytes. a and b are adjacent in "first" and in the + // destination; c leaves a five-row gap before it and a two-row tail. + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "first", .name = "a", .shape = .init(.{ 3, 8 }, .u16), .offset = 4 }, + .{ .file_uri = "first", .name = "b", .shape = .init(.{ 2, 8 }, .u16), .offset = 52 }, + .{ .file_uri = "second", .name = "c", .shape = .init(.{ 4, 8 }, .u16), .offset = 0 }, + .{ .file_uri = "second", .name = "plain", .shape = .init(.{ 4, 8 }, .u16), .offset = 64 }, + }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var paths: [2][std.Io.Dir.max_path_bytes]u8 = undefined; + var slots: [2]SourceSlot = undefined; + var opened: usize = 0; + defer for (slots[0..opened]) |*slot| slot.deinit(io); + for (&slots, &paths, [_][]const u8{ "first", "second" }, [_][]const u8{ &first_bytes, &second_bytes }) |*slot, *path, name, bytes| { + slot.* = .{ .uri = try writeTestFile(tmp.dir, io, name, bytes, path) }; + opened += 1; + } + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + loader.plan_config = .{ .block_size = block_size, .request_size = options.request_size }; + var outputs: [2]Buffer = undefined; + var destinations: [2]Destination = undefined; + var parts = [_]Part{ + .{ .source = &sources[0], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 0, .width = 48 } }, + .{ .source = &sources[1], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 48, .width = 32 } }, + .{ .source = &sources[2], .source_slot = &slots[1], .destination = &destinations[0], .lattice = .{ .dst_offset = 160, .width = 64 } }, + .{ .source = &sources[3], .source_slot = &slots[1], .destination = &destinations[1], .lattice = .{ .width = 64 } }, + }; + const plain_shape = shape.setDim(0, 4); + destinations = .{ + .{ .shape = shape, .grid = .init(platform, shape), .memory = options.memory, .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, + .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .memory = options.memory, .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, + }; + const host = destinations[0].memory.isHost(); + + const order = try Planner.sortedPartOrder(allocator, &parts); + defer allocator.free(order); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + var plans: [2]*Plan = undefined; + var plan_count: usize = 0; + var start: usize = 0; + while (start < order.len) : (plan_count += 1) { + const end = Planner.fileGroupEnd(&parts, order, start); + plans[plan_count] = try prepareTestPlan(arena.allocator(), &parts, order[start..end], .{ + .block_size = block_size, + .request_size = options.request_size, + .minimum_staging_gain = options.staging_gain, + }, options.alignment); + start = end; + } + try std.testing.expectEqual(@as(usize, 2), plan_count); + // The fill goes out once, with the plan of the destination's first + // part, as its last job. + const fill_kind: std.meta.Tag(Plan.Job.Kind) = if (host) .host_fill else .dma_fill; + try std.testing.expectEqual(fill_kind, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); + // Host jobs take no DMA event; DMA jobs one event per writer. + for (plans[0..plan_count], [_]usize{ 1, 0 }) |plan, fill_jobs| { + var writers: usize = 0; + var fills: usize = 0; + for (plan.jobs) |job| { + const kind = std.meta.activeTag(job.kind); + if (kind == fill_kind) fills += 1; + if (host) { + const read_kind: std.meta.Tag(Plan.Job.Kind) = if (options.alignment == 0) .host_direct else .host_bounce; + try std.testing.expect(kind == .host_fill or kind == read_kind); + } else { + try std.testing.expect(kind == .dma_read or kind == .dma_land or kind == .dma_fill); + } + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| writers += @popCount(transfer.writer_mask); + } + try std.testing.expectEqual(fill_jobs, fills); + try std.testing.expectEqual(if (host) 0 else writers, plan.events.len); + } + + const zeros = try allocator.alloc(u8, 256); + defer allocator.free(zeros); + @memset(zeros, 0); + var replays: [2]Replay = undefined; + var initialized: usize = 0; + defer for (replays[0..initialized]) |*replay| replay.deinit(allocator); + for (&replays, &destinations) |*replay, *destination| { + replay.* = try .init(allocator, platform.placedShape(destination.shape).packedShape()); + initialized += 1; + } + var cross_part_piece = false; + for (plans[0..plan_count]) |plan| { + const file_bytes: []const u8 = if (plan.source_slot == &slots[0]) &first_bytes else &second_bytes; + for (plan.jobs) |job| { + const fill = job.kind == .dma_fill or job.kind == .host_fill; + const blocks = if (host or fill) &.{} else try jobBlocks(allocator, loader, plan, job); + defer allocator.free(blocks); + for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { + // Every transfer stays inside one block (`Plan.Transfer`). + try std.testing.expect(transfer.landing % block_size + transfer.len <= block_size); + switch (job.kind) { + .host_direct, .host_bounce => |read| try std.testing.expect(transfer.landing + transfer.len <= read.len), + else => try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size), + } + const bytes = if (fill) fill: { + try std.testing.expectEqual(@as(usize, 0), transfer.landing); + break :fill zeros[0..transfer.len]; + } else if (host) read: { + // A host job's landings address its read, which + // starts at the job's file offset. + const file_offset = job.file_offset + transfer.landing; + if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read file_bytes[file_offset..][0..transfer.len]; + } else read: { + // a's last byte and b's first one in the same piece, + // where a read's blocks hold the file in order. + const file_offset = job.file_offset + transfer.landing; + if (job.kind == .dma_read and plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; + break :read blocks[transfer.landing..][0..transfer.len]; + }; + const index: usize = if (transfer.destination == &destinations[0]) 0 else 1; + try replays[index].write(transfer.writer_mask, transfer.destination_offset, bytes); + } + } + } + try std.testing.expectEqual(options.cross_part_piece, cross_part_piece); + + var global: [256]u8 = @splat(0); + @memcpy(global[0..80], first_bytes[4..84]); + @memcpy(global[160..224], second_bytes[0..64]); + try replays[0].expect(allocator, &global); + try replays[1].expect(allocator, second_bytes[64..128]); + } + + /// A DMA job's blocks taken end to end once its worker has read it: + /// the read in file order, or as its worker lands a landing job + /// (`landJob`). + fn jobBlocks(allocator: std.mem.Allocator, loader: *Loader, plan: *const Plan, job: Plan.Job) ![]u8 { + const block_size = loader.plan_config.block_size; + const bytes = try allocator.alloc(u8, job.blockLen(block_size) * block_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const land = switch (job.kind) { + .dma_read => |read| { + // The end of the file cuts a widened read's padding. + var buffers = [_][]u8{bytes[0..read.len]}; + try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); + return bytes; + }, + .dma_land => |land| land, + .dma_fill, .host_direct, .host_bounce, .host_fill => unreachable, + }; + // A bounce buffer as large as any of the job's reads. + var longest: usize = land.len; + for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| longest = @max(longest, extent.len); + const blocks = try allocator.alloc(host_memory.Block, job.blockLen(block_size)); + defer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = bytes[index * block_size ..][0..block_size]; + var scratch: ReadRequest.Scratch = try .init(allocator, Planner.blockCount(longest, block_size), block_size); + defer scratch.deinit(); + try landJob(loader, plan, land, blocks, &scratch); + return bytes; + } + + /// One destination's device buffers, written by replayed transfers. + const Replay = struct { + shape: Shape, + placement: Sharding.Placement, + writer_size: usize, + bytes: []u8, + written: []bool, + + fn init(allocator: std.mem.Allocator, placed_shape: Shape) !Replay { + const placement: Sharding.Placement = try .init(&placed_shape); + const writers = placement.mesh().?.devicesInCanonicalOrder().len; + const writer_size = placement.localShape(placed_shape).byteSize(); + const bytes = try allocator.alloc(u8, writers * writer_size); + errdefer allocator.free(bytes); + @memset(bytes, 0xaa); + const written = try allocator.alloc(bool, bytes.len); + @memset(written, false); + return .{ .shape = placed_shape, .placement = placement, .writer_size = writer_size, .bytes = bytes, .written = written }; + } + + fn deinit(self: *Replay, allocator: std.mem.Allocator) void { + allocator.free(self.written); + allocator.free(self.bytes); + } + + /// Each byte of each device at most once. + fn write(self: *Replay, writer_mask: u64, destination_offset: usize, source: []const u8) !void { + var mask = writer_mask; + while (mask != 0) { + const writer: usize = @intCast(@ctz(mask)); + mask &= mask - 1; + try std.testing.expect(destination_offset + source.len <= self.writer_size); + const at = writer * self.writer_size + destination_offset; + for (self.written[at..][0..source.len]) |*flag| { + try std.testing.expect(!flag.*); + flag.* = true; + } + @memcpy(self.bytes[at..][0..source.len], source); + } + } + + /// Every device holds its slice of the rank-2 `global`, entirely. + fn expect(self: *const Replay, allocator: std.mem.Allocator, global: []const u8) !void { + for (self.written) |flag| try std.testing.expect(flag); + const shape = self.shape; + const element_bytes = shape.dtype().sizeOf(); + const row_bytes: usize = @intCast(shape.dim(1) * element_bytes); + const expected = try allocator.alloc(u8, self.writer_size); + defer allocator.free(expected); + for (0..self.placement.mesh().?.devicesInCanonicalOrder().len) |writer| { + const slices = self.placement.slices(writer, shape); + const rows = slices.get(0); + const columns = slices.get(1); + const len: usize = @intCast(columns.size * element_bytes); + var cursor: usize = 0; + for (@intCast(rows.start)..@intCast(rows.start + rows.size)) |row| { + @memcpy(expected[cursor..][0..len], global[row * row_bytes + @as(usize, @intCast(columns.start * element_bytes)) ..][0..len]); + cursor += len; + } + try std.testing.expectEqualSlices(u8, expected, self.bytes[writer * self.writer_size ..][0..self.writer_size]); + } + } + }; +}; + +test "composite destinations place each part and zero-fill the rest of every device" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const shape = Shape.init(.{ .row = 16, .col = 8 }, .u16); + // Two by two devices, then two by four. + for ([_]u32{ 4, 8 }) |device_count| { + var platform = Platform.init(allocator, io, .cpu, .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = device_count }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const physical = &platform.physical_mesh; + const logical: Sharding.LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + const halves: Sharding.Mesh = try .init("halves", physical, logical, .parseBindings(.{ .model = .link_x })); + const quarters: Sharding.Mesh = if (device_count == 4) + try .init("quarters", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })) + else + try .init("quarters", physical, logical, .parseBindings(.{ .model = .link_y })); + const all: Sharding.Mesh = try .init("all", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })); + const scenarios = .{ + // Two row shards, each on half the devices: b and c straddle no + // boundary, the gap does. + .{ &halves, .{ .row = .model }, true }, + // Four row shards: b, c and the gap straddle shards. + .{ &quarters, .{ .row = .model }, true }, + // Every row of every source split across two devices, then + // across every device. + .{ &halves, .{ .col = .model }, false }, + .{ &all, .{ .col = .model }, false }, + // Every byte on every device: one span, so a and b merge. + .{ &all, .{}, true }, + }; + inline for (scenarios) |scenario| { + const placed = shape.withPartitioning(scenario[0], scenario[1]); + // 64-byte blocks hold a's tail and b's head together; 16-byte ones + // cut every row apart and every fill piece to one block. + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .cross_part_piece = false }); + // Staged wherever it saves a submission: the column-sharded jobs + // land grouped by device, in reads exact or widened. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .alignment = alignment, .staging_gain = 1, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 48, .alignment = alignment, .staging_gain = 1, .cross_part_piece = false }); + } + // Host jobs, read in place or through blocks when the reads are + // widened, are cut by blocks the same way. + for ([_]usize{ 0, 8 }) |alignment| { + try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = scenario[2] }); + try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = false }); + } + } + } +} + +// Scheduler tests. + +/// A plan of `job_count` unit jobs (`file_offset` = index), each with one +/// block and one event slot, and no transfers, in `arena`. +fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { + const jobs = try arena.alloc(Plan.Job, job_count); + const requests = try arena.alloc(ReadRequest, job_count); + @memset(requests, ReadRequest.idle); + const events = try arena.alloc(Dma.EventContext, job_count); + for (jobs, 0..) |*job, index| job.* = .{ + .file_offset = index, + .transfer_start = 0, + .transfer_len = 0, + .kind = .{ .dma_read = .{ .len = 1, .minimum_len = 1 } }, + }; + const plan = try arena.create(Plan); + plan.* = .{ + .source_slot = undefined, + .jobs = jobs, + .transfers = &.{}, + .requests = requests, + .events = events, + .source_bytes = job_count, + }; + return plan; +} + +/// Publishes a plan of `job_count` unit jobs into an open batch. +fn publishTestPlan(scheduler: *Scheduler, batch: *Batch, job_count: usize) !void { + try scheduler.publish(std.testing.io, batch, try testPlan(batch.arena.allocator(), job_count)); +} + +/// A one-file submission's sequence: publish, then drop the sentinel. +fn publishTestBatch(scheduler: *Scheduler, job_count: usize) !*Batch { + const batch = try Batch.create(std.testing.allocator, std.testing.io, 0); + errdefer batch.destroy(); + try publishTestPlan(scheduler, batch, job_count); + batch.releaseUnits(1); + return batch; +} + +/// The jobs of the queued plans no worker claimed yet. +fn unclaimedJobs(scheduler: *const Scheduler) usize { + var jobs: usize = 0; + for (0..scheduler.queue.len) |index| { + const plan = scheduler.queue.at(index); + jobs += plan.jobs.len - plan.cursor; + } + return jobs; +} + +test "scheduler publishes a submission one file at a time and claims the files in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + // Submitted out of file order: the sort groups "a" before "b". + var sources = [_]safetensors.Tensor{ + .{ .file_uri = "b", .name = "b0", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 4 }, + .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 0 }, + }; + var slots = [_]SourceSlot{ + .{ .uri = "a" }, + .{ .uri = "b" }, + }; + var outputs: [sources.len]Buffer = undefined; + var destinations: [sources.len]Destination = undefined; + var parts: [sources.len]Part = undefined; + initPlainParts(platform, &sources, &.{ &slots[1], &slots[0], &slots[0] }, &outputs, &destinations, &parts); + + var scheduler: Scheduler = .init(allocator); + defer scheduler.deinit(); + const batch = try Batch.create(allocator, io, 0); + try Planner.publishFiles(&scheduler, io, batch, &parts, .{ + .block_size = 4, + .request_size = 4, + }, fixedReads(0)); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items.len); + try std.testing.expectEqual(@as(usize, 2), batch.plans.items[0].jobs.len); + try std.testing.expectEqual(@as(usize, 1), batch.plans.items[1].jobs.len); + try std.testing.expectEqual(@as(usize, 3), unclaimedJobs(&scheduler)); + try std.testing.expect(batch.published_at != null); + + // File a's jobs in offset order, then file b's. + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[0]); + try std.testing.expectEqual(@as(u64, 4), claim.plan.jobs[claim.index].file_offset); + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan.source_slot == &slots[1]); + try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); + // Both plans left the queue with their last claim; the submission is + // still open until its sentinel goes. + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + batch.releaseUnits(1); + try std.testing.expect(!batch.done.isSet()); + batch.releaseUnits(3); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); +} + +test "fifo scheduler claims plan by plan in publish order, and a done batch leaves while a later plan is queued" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // An open submission's first plan, a sealed batch, then the open + // submission's second plan: an open submission holds no place of its + // own. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 1); + const sealed = try publishTestBatch(&scheduler, 2); + try publishTestPlan(&scheduler, open, 1); + try std.testing.expectEqual(@as(usize, 4), unclaimedJobs(&scheduler)); + + var claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[0] and claim.index == 0); + for (0..2) |index| { + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == sealed.plans.items[0] and claim.index == index); + } + // The sealed batch left the queue with its last claim, so it completes + // and goes away while the open one's second plan is still queued. + sealed.releaseUnits(2); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expectEqual(@as(usize, 1), scheduler.queue.len); + + claim = scheduler.claim(io).?; + try std.testing.expect(claim.plan == open.plans.items[1] and claim.index == 0); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + // Claimed out, the open submission completes only once its sentinel + // goes. + open.releaseUnits(2); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler failure retires the unclaimed jobs of every queued plan and refuses later publishes" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + // A batch without jobs is never queued and completes at publish. + const empty = try publishTestBatch(&scheduler, 0); + try std.testing.expect(empty.done.isSet()); + empty.destroy(); + // An open submission of two plans, then a sealed batch. + const open = try Batch.create(std.testing.allocator, io, 0); + try publishTestPlan(&scheduler, open, 2); + try publishTestPlan(&scheduler, open, 3); + const sealed = try publishTestBatch(&scheduler, 3); + try std.testing.expectEqual(@as(usize, 8), unclaimedJobs(&scheduler)); + try std.testing.expectEqual(@as(usize, 6), open.remaining.load(.acquire)); + + // The claim keeps its unit and the open submission its sentinel; every + // other job is retired. + try std.testing.expect(scheduler.claim(io).?.plan.batch == open); + scheduler.fail(io); + try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); + try std.testing.expectEqual(@as(usize, 2), open.remaining.load(.acquire)); + try std.testing.expect(sealed.done.isSet()); + sealed.destroy(); + try std.testing.expect(scheduler.claim(io) == null); + try std.testing.expect(!scheduler.waitForWork(io)); + // The submission goes on: its next plan is refused, and the sentinel + // drop leaves only the claim to complete it. + try std.testing.expectError(error.Canceled, publishTestPlan(&scheduler, open, 1)); + open.releaseUnits(1); + try std.testing.expect(!open.done.isSet()); + open.releaseUnits(1); + try std.testing.expect(open.done.isSet()); + open.destroy(); +} + +test "fifo scheduler wakes waiting workers on publish and releases them on failure" { + const WaitResult = enum(u8) { waiting, work, stopped }; + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + + var woke: std.atomic.Value(WaitResult) = .init(.waiting); + var group: std.Io.Group = .init; + const Waiter = struct { + fn run(scheduler_: *Scheduler, io_: std.Io, woke_: *std.atomic.Value(WaitResult)) void { + woke_.store(if (scheduler_.waitForWork(io_)) .work else .stopped, .release); + } + }; + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + const batch = try publishTestBatch(&scheduler, 1); + try group.await(io); + try std.testing.expectEqual(WaitResult.work, woke.load(.acquire)); + _ = scheduler.claim(io).?; + batch.releaseUnits(1); + batch.destroy(); + + woke.store(.waiting, .release); + try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); + try io.sleep(.fromMilliseconds(5), .awake); + try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); + scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(WaitResult.stopped, woke.load(.acquire)); +} + +test "fifo scheduler concurrent claims across two batches return each job once" { + const io = std.testing.io; + var scheduler: Scheduler = .init(std.testing.allocator); + defer scheduler.deinit(); + const batches = [_]*Batch{ + try publishTestBatch(&scheduler, 16), + try publishTestBatch(&scheduler, 16), + }; + var seen: std.atomic.Value(u64) = .init(0); + var claim_count: std.atomic.Value(usize) = .init(0); + var duplicate: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + for (0..8) |_| try group.concurrent(io, struct { + fn run( + scheduler_: *Scheduler, + batches_: []const *Batch, + seen_: *std.atomic.Value(u64), + claim_count_: *std.atomic.Value(usize), + duplicate_: *std.atomic.Value(bool), + ) void { + while (scheduler_.claim(std.testing.io)) |claim| { + const base: u64 = if (claim.plan.batch == batches_[0]) 0 else 16; + const mask = @as(u64, 1) << @intCast(base + claim.plan.jobs[claim.index].file_offset); + if (seen_.fetchOr(mask, .acq_rel) & mask != 0) duplicate_.store(true, .release); + _ = claim_count_.fetchAdd(1, .monotonic); + } + } + }.run, .{ &scheduler, &batches, &seen, &claim_count, &duplicate }); + try group.await(io); + try std.testing.expectEqual(std.math.maxInt(u32), @as(u32, @truncate(seen.load(.acquire)))); + try std.testing.expectEqual(@as(usize, 32), claim_count.load(.acquire)); + try std.testing.expect(!duplicate.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&scheduler)); + for (batches) |batch| { + batch.releaseUnits(16); + try std.testing.expect(batch.done.isSet()); + batch.destroy(); + } +} + +// DMA pump and transfer target tests. + +test "device pump admits by bytes and by pieces" { + var pump: Dma.DevicePump = .{}; + try std.testing.expect(pump.hasRoom(8)); + // The piece that crosses the budget was admitted; the next is not. + pump.active_bytes = 8; + try std.testing.expect(!pump.hasRoom(8)); + pump.active_bytes = 7; + try std.testing.expect(pump.hasRoom(8)); + pump.active_pieces = max_dma_pieces_per_device; + try std.testing.expect(!pump.hasRoom(8)); +} + +test "the submission that completes a target's bytes carries the last flag" { + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 100 }; + try std.testing.expect(!target.fullySubmitted()); + // Pieces arrive in any order; only the byte total matters. + try std.testing.expect(!target.nextIsLast(20)); + target.noteSubmitted(20); + try std.testing.expect(!target.nextIsLast(30)); + target.noteSubmitted(30); + try std.testing.expect(!target.nextIsLast(10)); + try std.testing.expect(target.nextIsLast(50)); + try std.testing.expect(!target.fullySubmitted()); + target.noteSubmitted(50); + try std.testing.expect(target.fullySubmitted()); +} + +// Request, credit and batch lifecycle tests. + +/// A single-device loader without PJRT, workers or pinned arenas: enough +/// for request, credit and batch lifecycle tests, with 64-byte blocks cut +/// as `create` cuts them, the zero block then `slots` credits of one block. +/// `platform` and `workspace` are its only undefined fields: what these +/// tests run reaches the platform only through a non-null PJRT event +/// (`Dma.EventContext.destroyEvent`). Must not move after `init`. +const TestLoader = struct { + memory: []u8, + pumps: [1]Dma.DevicePump = .{.{}}, + loader: Loader, + + fn init(self: *TestLoader, slots: usize) !void { + const allocator = std.testing.allocator; + const memory = try allocator.alloc(u8, (slots + 1) * 64); + errdefer allocator.free(memory); + const blocks = try allocator.alloc(host_memory.Block, slots + 1); + errdefer allocator.free(blocks); + for (blocks, 0..) |*block, index| block.* = memory[index * 64 ..][0..64]; + @memset(blocks[0], 0); + self.* = .{ .memory = memory, .loader = undefined }; + self.loader = .{ + .allocator = allocator, + .io = std.testing.io, + .platform = undefined, + .workspace = undefined, + .blocks = blocks, + .scheduler = .init(allocator), + .read_permits = .{ .permits = 1 }, + .read_width = 1, + .credits = try .init(allocator, blocks[1..], 1), + .pumps = &self.pumps, + .dma_budget_bytes = 64, + .created_at = .now(std.testing.io, .awake), + .plan_config = .{ .block_size = 64, .request_size = 64 }, + .reads = fixedReads(0), + }; + } + + fn deinit(self: *TestLoader) void { + const allocator = std.testing.allocator; + self.loader.scheduler.deinit(); + self.pumps[0].queue.deinit(allocator); + self.loader.credits.deinit(allocator); + allocator.free(self.loader.blocks); + allocator.free(self.memory); + } +}; + +test "a callback failing while a piece is queued drains it and completes the batch" { + const io = std.testing.io; + const allocator = std.testing.allocator; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // One request whose only piece is queued behind a DMA slot that a failed + // callback is about to free. + const batch = try publishTestBatch(&loader.scheduler, 1); + const request = loader.claimRequest().?; + var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 64 }; + _ = request.pending.fetchAdd(1, .acq_rel); + try fixture.pumps[0].queue.pushBack(allocator, .{ + .target = &target, + .request = request, + .data = loader.credits.slotBlocks(request.slot)[0], + .destination_offset = 0, + }); + fixture.pumps[0].active_bytes = 64; + fixture.pumps[0].active_pieces = 1; + request.release(); + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 1), loader.credits.inUse(io)); + _ = loader.first_error.set(error.Unknown); + + Dma.eventCompleted(loader, 0, 64); + try std.testing.expectEqual(@as(usize, 0), fixture.pumps[0].active_pieces); + try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expect(batch.done.isSet()); + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "batch completes when every claimed request completes" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(3); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 3); + try std.testing.expectEqual(@as(usize, 3), batch.remaining.load(.acquire)); + var requests: [3]*ReadRequest = undefined; + for (&requests) |*request| request.* = loader.claimRequest().?; + try std.testing.expect(loader.scheduler.claim(io) == null); + try std.testing.expectEqual(@as(usize, 3), loader.credits.inUse(io)); + + for (requests, 0..) |request, index| { + try std.testing.expect(!batch.done.isSet()); + try std.testing.expectEqual(requests.len - index, batch.remaining.load(.acquire)); + request.release(); + } + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), batch.remaining.load(.acquire)); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + // Each claim took its job's slot of the plan's request array. + for (requests, batch.plans.items[0].requests) |request, *slot| { + try std.testing.expect(request == slot); + } + + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "a worker waiting for a credit when the loader fails gets one back and stops" { + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(1); + defer fixture.deinit(); + const loader = &fixture.loader; + + // A request in flight holds the only credit; a worker that saw the + // second job waits on the credits. + const batch = try publishTestBatch(&loader.scheduler, 2); + const request = loader.claimRequest().?; + try loader.worker_group.concurrent(io, Loader.workerMain, .{loader}); + while (loader.credits.condition.state.load(.monotonic).waiters == 0) io.sleep(.fromMilliseconds(1), .awake) catch {}; + + // The failure retires the second job and closes nothing: the request + // gives its credit back as it ends, the worker claims nothing with it, + // and `destroy`'s shutdown stops it. + loader.recordError(error.InputOutput, "injected source failure", .{}); + try std.testing.expect(!batch.done.isSet()); + request.release(); + loader.stopWorkers(); + try std.testing.expect(batch.done.isSet()); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(@as(usize, 0), batch.plans.items[0].requests[1].pending.load(.acquire)); + Dma.retireBatch(loader, batch); + batch.destroy(); +} + +test "retired events are destroyed by the next pump or by a batch's retirement" { + // Without PJRT, contexts with a null event stand in for destroyed ones; + // the list mechanics are the subject. + const io = std.testing.io; + var fixture: TestLoader = undefined; + try fixture.init(2); + defer fixture.deinit(); + const loader = &fixture.loader; + + const batch = try publishTestBatch(&loader.scheduler, 2); + const requests = [_]*ReadRequest{ + loader.claimRequest().?, + loader.claimRequest().?, + }; + const plan = batch.plans.items[0]; + plan.events_used.store(2, .release); + for (plan.events, requests) |*ctx, request| ctx.* = .{ + .request = request, + .pjrt_event = null, + .device_index = 0, + .len = 64, + }; + // Two callbacks fired: the next pump destroys both, newest first. + Dma.retireEvent(loader, &plan.events[0]); + Dma.retireEvent(loader, &plan.events[1]); + const device_pump = &fixture.pumps[0]; + try std.testing.expect(device_pump.retired == &plan.events[1]); + try std.testing.expect(plan.events[1].next_retired == &plan.events[0]); + device_pump.mutex.lockUncancelable(io); + device_pump.destroyRetired(); + device_pump.mutex.unlock(io); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(plan.events[1].next_retired == null); + + // A callback that fires after the last pump leaves its context linked; + // the batch's retirement destroys it before the batch is freed, with + // the retired contexts of a batch still in flight. + var other: Dma.EventContext = .{ .request = undefined, .pjrt_event = null, .device_index = 0, .len = 64 }; + Dma.retireEvent(loader, &other); + Dma.retireEvent(loader, &plan.events[0]); + for (requests) |request| request.release(); + try std.testing.expect(batch.done.isSet()); + Dma.retireBatch(loader, batch); + try std.testing.expect(device_pump.retired == null); + try std.testing.expect(plan.events[0].next_retired == null); + try std.testing.expect(other.next_retired == null); + batch.destroy(); +} + +test "overlapping batches complete under concurrent claims and retirement" { + const io = std.testing.io; + const worker_count = 4; + var fixture: TestLoader = undefined; + try fixture.init(worker_count); + defer fixture.deinit(); + const loader = &fixture.loader; + + const Worker = struct { + fn run(loader_: *Loader) void { + while (loader_.scheduler.waitForWork(loader_.io)) if (loader_.claimRequest()) |request| request.release(); + } + }; + var group: std.Io.Group = .init; + defer { + loader.scheduler.fail(io); + group.await(io) catch {}; + } + for (0..worker_count) |_| try group.concurrent(io, Worker.run, .{loader}); + + // Three batches in flight at a time, awaited newest first: a batch is + // retired while later ones are still being claimed. + for (0..70) |round| { + var batches: [3]*Batch = undefined; + var job_counts: [3]usize = undefined; + for (&batches, &job_counts, 0..) |*batch, *job_count, offset| { + job_count.* = (round * 3 + offset) % 7; + batch.* = try publishTestBatch(&loader.scheduler, job_count.*); + } + var index = batches.len; + while (index > 0) { + index -= 1; + const batch = batches[index]; + batch.done.waitUncancelable(io); + Dma.retireBatch(loader, batch); + batch.destroy(); + } + try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&loader.scheduler)); + } + // Every request released its credit before its batch completed; the + // workers only hold one while a claim is in progress. + loader.scheduler.fail(io); + try group.await(io); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + +// Source read tests. + +test "readFilePositionalAllV resumes short reads and stops at minimum" { + const OneByteIo = struct { + const file_end = 11; + + /// Reads one byte per call, the low byte of its offset, from a file + /// that ends at `file_end`, and counts the calls in `userdata`. + fn fileReadPositional(userdata: ?*anyopaque, _: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const calls: *usize = @ptrCast(@alignCast(userdata.?)); + calls.* += 1; + if (offset >= file_end) return 0; + for (data) |buffer| { + if (buffer.len == 0) continue; + buffer[0] = @truncate(offset); + return 1; + } + return 0; + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = OneByteIo.fileReadPositional; + var calls: usize = 0; + const io: std.Io = .{ .userdata = &calls, .vtable = &vtable }; + const file: std.Io.File = .{ .handle = 0, .flags = .{ .nonblocking = false } }; + + var output: [5]u8 = @splat(0xaa); + var buffers = [_][]u8{ output[0..1], output[1..1], output[1..3], output[3..5] }; + try std.testing.expectEqual(@as(u64, 4), try readFilePositionalAllV(io, file, &buffers, 7, 4)); + try std.testing.expectEqual(@as(usize, 4), calls); + try std.testing.expectEqualSlices(u8, &.{ 7, 8, 9, 10, 0xaa }, &output); + + var past_end = [_][]u8{&output}; + try std.testing.expectError(error.UnexpectedEndOfFile, readFilePositionalAllV(io, file, &past_end, 7, output.len)); +} + +// LazyOnce tests. + +test "lazy initialization preserves its finite error across repeated callers" { + const Context = struct { + fn init(calls: *usize) error{FileNotFound}!u32 { + calls.* += 1; + return error.FileNotFound; + } + }; + var once: LazyOnce(u32) = .{}; + var calls: usize = 0; + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); + try std.testing.expectEqual(1, calls); +} + +// Whole-loader tests. + +/// Submits `specs`, which must fail with `expected`; a batch that went out +/// anyway is awaited before the test fails. +fn expectSubmitError(loader: *Loader, specs: []const LoadSpec, expected: anyerror) !void { + const batch = loader.submit(specs, .none) catch |err| return std.testing.expectEqual(expected, err); + loader.awaitBatch(batch) catch {}; + return error.TestUnexpectedResult; +} + +test "loader releases the pinned set when its workers cannot start" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // The testing io, but no worker starts: `create` fails after the carve, + // the credits and the pumps, and unwinds every one. + var vtable = std.testing.io.vtable.*; + vtable.groupConcurrent = std.Io.failingGroupConcurrent; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const result: anyerror!void = if (Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off })) |loader| unexpected: { + loader.destroy(); + break :unexpected {}; + } else |err| err; + try std.testing.expectError(error.ConcurrencyUnavailable, result); +} + +test "loader failures clean up before publication, after publication and during reading" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + var later_buffer: [1024]u8 = undefined; + const later_path = try writeTestFile(tmp.dir, io, "weights.bin.later", &contents, &later_buffer); + const sources = [_]safetensors.Tensor{ + .{ .file_uri = path, .name = "first", .shape = .init(.{4}, .u8), .offset = 0 }, + .{ .file_uri = later_path, .name = "later", .shape = .init(.{5}, .u8), .offset = 0 }, + }; + const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; + + // Device memory goes through blocks and DMA; host memory is written in + // place by the workers, into views of it for `host_mapped`. All clean + // up the same way. + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { + // Planning allocates from the scheduler's allocator, whose + // allocations fail in turn: the first file's planning fails before + // any plan is published, then the later file's after the first + // file's plan was. Past the last, planning succeeds and the read + // fails: the later source ends after four of five bytes. + var fail_index: usize = 0; + while (true) : (fail_index += 1) { + var outputs: [2]Buffer = @splat(.{ + ._platform = platform, + ._shape = sources[0].shape, + ._shards = .empty, + }); + defer for (&outputs) |*output| output.deinit(); + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = fail_index }); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.scheduler.allocator = failing.allocator(); + const specs = [_]LoadSpec{ + .{ .shape = sources[0].shape, .memory = memory, .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = sources[1].shape, .memory = memory, .output = &outputs[1], .parts = parts[1..2] }, + }; + + if (loader.submit(&specs, .none)) |batch| { + try std.testing.expectError(error.UnexpectedEndOfFile, loader.awaitBatch(batch)); + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for + // a claim that finds nothing: every credit is free once the + // workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + + // Each failure fails the loader: a later submission gets it. + try std.testing.expectEqual(@as(usize, 1), loader.batch_count); + const expected: anyerror = if (failing.has_induced_failure) error.OutOfMemory else error.UnexpectedEndOfFile; + try std.testing.expectError(expected, loader.checkOpen()); + try expectSubmitError(loader, specs[0..1], expected); + if (!failing.has_induced_failure) break; + } + } +} + +test "exactly the width of reads are in flight, and a read failing mid-load leaves nothing held" { + const allocator = std.testing.allocator; + const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, std.testing.io); + // Every source read holds for a moment and is counted; the one numbered + // `fail_at` fails. + const Reads = struct { + var in_flight: std.atomic.Value(usize) = .init(0); + var peak: std.atomic.Value(usize) = .init(0); + var started: std.atomic.Value(usize) = .init(0); + var fail_at: usize = std.math.maxInt(usize); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + _ = peak.fetchMax(in_flight.fetchAdd(1, .acq_rel) + 1, .acq_rel); + defer _ = in_flight.fetchSub(1, .acq_rel); + std.testing.io.sleep(.fromMilliseconds(5), .awake) catch {}; + if (started.fetchAdd(1, .acq_rel) == fail_at) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + // Sixteen tensors with gaps between them: a read, and a job, each. + const tensors = 16; + var contents: [tensors * 8]u8 = undefined; + for (&contents, 0..) |*byte, index| byte.* = @truncate(index); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", &contents, &path_buffer); + var sources: [tensors]safetensors.Tensor = undefined; + var parts: [tensors]LoadPart = undefined; + var specs: [tensors]LoadSpec = undefined; + var outputs: [tensors]Buffer = undefined; + for (&sources, &parts, &specs, &outputs, 0..) |*source, *part, *spec, *output, index| { + source.* = .{ .file_uri = path, .name = "t", .shape = .init(.{4}, .u8), .offset = index * 8 }; + part.* = .whole(source); + spec.* = .{ .shape = source.shape, .output = output, .parts = part[0..1] }; + } + + for ([_]?usize{ null, 5 }) |fail_at| { + for (&outputs) |*output| output.* = .{ ._platform = platform, ._shape = sources[0].shape, ._shards = .empty }; + defer for (&outputs) |*output| output.deinit(); + // `destroy` returns: no worker waits for a credit or a permit. + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + try std.testing.expectEqual(@as(usize, 2), loader.read_width); + Reads.in_flight = .init(0); + Reads.peak = .init(0); + Reads.started = .init(0); + Reads.fail_at = fail_at orelse std.math.maxInt(usize); + const batch = try loader.submit(&specs, .none); + if (fail_at == null) { + try loader.awaitBatch(batch); + // Three workers, two permits. + try std.testing.expectEqual(@as(usize, tensors), Reads.started.load(.acquire)); + try std.testing.expectEqual(loader.read_width, Reads.peak.load(.acquire)); + try testing.expectShards(allocator, std.testing.io, &outputs[tensors - 1], contents[(tensors - 1) * 8 ..][0..4], .default); + } else { + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The reads stopped at the failure. + try std.testing.expect(Reads.started.load(.acquire) < tensors); + } + try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); + // A worker the batch woke may still hold the credit it took for a + // claim that finds nothing: every credit is free once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); + } +} + +test "a default-memory output a failed load left unfinished reports an error on every device" { + const allocator = std.testing.allocator; + // On the accelerator when it has two devices: each plugin's transfer + // manager errors the buffers it leaves unfinished. + var devices: testing.Devices = undefined; + try devices.init(allocator, std.testing.io, 2); + defer devices.deinit(); + const platform = devices.platform; + // The read reaching the last request fails. + const Reads = struct { + var fail_from: u64 = std.math.maxInt(u64); + + fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + if (offset >= fail_from) return error.InputOutput; + return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); + } + }; + var vtable = std.testing.io.vtable.*; + vtable.fileReadPositional = Reads.readPositional; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + + // A replicated tensor of three requests, a job each: no device's last + // piece goes out. + const request_size = loader.plan_config.request_size; + const contents = try allocator.alloc(u8, 3 * request_size); + defer allocator.free(contents); + testing.pattern(contents, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", contents, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "t", .shape = .init(.{contents.len}, .u8), .offset = 0 }; + var part: LoadPart = .whole(&source); + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const spec: LoadSpec = .{ .shape = source.shape, .output = &output, .parts = (&part)[0..1] }; + + Reads.fail_from = 2 * request_size; + defer Reads.fail_from = std.math.maxInt(u64); + const batch = try loader.submit((&spec)[0..1], .none); + try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); + // The failing job created the buffers before its read. Each one is + // ready, with an error, once the batch is retired: a success would + // hand out a buffer missing its last request, a pending one would hang + // its first user. + const placed = platform.placedShape(source.shape); + try std.testing.expectEqual(placed._sharding.mesh.?.devicesInCanonicalOrder().len, output._shards.len); + for (output._shards.constSlice()) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try std.testing.expect(event.isReady(platform.pjrt_api)); + const err = event.getEventError(platform.pjrt_api) orelse return error.TestUnexpectedResult; + err.deinit(platform.pjrt_api); + } +} + +test "whole tensors load on four devices, sharded by rows or columns, replicated, or joined from two files" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 16 per device; rows cut into four pieces of 8 + // bytes, one per device; a table on every device; and a tensor whose + // rows continue in a second file. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var table: [3000 * 4]u8 = undefined; + var tail: [64 * 32]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&table, 3); + testing.pattern(&tail, 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "table", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &table }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "tail", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &tail }, + }); + var joined: [rows.len + tail.len]u8 = undefined; + @memcpy(joined[0..rows.len], &rows); + @memcpy(joined[rows.len..], &tail); + const parts = [_]LoadPart{ + .whole(registry.tensors.getPtr("rows").?), + .whole(registry.tensors.getPtr("cols").?), + .whole(registry.tensors.getPtr("table").?), + // rows again, then tail after it: the first file's run overlaps. + .whole(registry.tensors.getPtr("rows").?), + .{ .source = registry.tensors.getPtr("tail").?, .lattice = .{ .dst_offset = rows.len, .width = tail.len } }, + }; + var outputs: [4]Buffer = undefined; + const specs = [_]LoadSpec{ + .{ .shape = Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[0], .parts = parts[0..1] }, + .{ .shape = Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), .output = &outputs[1], .parts = parts[1..2] }, + .{ .shape = Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{}), .output = &outputs[2], .parts = parts[2..3] }, + .{ .shape = Shape.init(.{ .row = 128, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[3], .parts = parts[3..5] }, + }; + const globals = [_][]const u8{ &rows, &cols, &table, &joined }; + + // Blocks of 4 KiB and requests of two: the first file's run is cut into + // jobs, and the jobs' transfers at block ends. + const loader = try Loader.create(allocator, io, devices.platform, .{ + .read_parallelism = 2, + .read_hints = .{ .read_chunk_size = 8192 }, + .dma_block_size = 4096, + .direct_io = .off, + }); + defer loader.destroy(); + const batch = try loader.submit(&specs, .none); + var jobs: usize = 0; + for (batch.plans.items) |plan| jobs += plan.jobs.len; + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + // 46,816 bytes in the first file, 2,048 in the second. + try std.testing.expectEqual(@as(usize, 6 + 1), jobs); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); +} + +test "a submit the loader fails between its open check and its first publish returns that failure" { + const io = std.testing.io; + const platform = Platform.init(std.testing.allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(std.testing.allocator, io); + // Fails the loader in the submitting task's next allocation: after + // `submit`'s open check and before its first publish, where a worker of + // an older batch can fail it and the scheduler then answers `Canceled`. + const FailNextAllocation = struct { + parent: std.mem.Allocator, + loader: *Loader = undefined, + thread: std.Thread.Id = undefined, + armed: std.atomic.Value(bool) = .init(false), + + fn arm(self: *@This(), loader: *Loader) void { + self.loader = loader; + self.thread = std.Thread.getCurrentId(); + self.armed.store(true, .release); + } + + fn allocator(self: *@This()) std.mem.Allocator { + return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + // The workers allocate on threads of their own. + if (self.armed.load(.acquire) and self.thread == std.Thread.getCurrentId()) { + self.armed.store(false, .monotonic); + self.loader.recordError(error.UnexpectedEndOfFile, "an older batch's read", .{}); + } + return self.parent.rawAlloc(len, alignment, ret_addr); + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawResize(memory, alignment, new_len, ret_addr); + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + const self: *@This() = @ptrCast(@alignCast(ctx)); + return self.parent.rawRemap(memory, alignment, new_len, ret_addr); + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *@This() = @ptrCast(@alignCast(ctx)); + self.parent.rawFree(memory, alignment, ret_addr); + } + }; + var failing: FailNextAllocation = .{ .parent = std.testing.allocator }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &.{ 1, 2, 3, 4 }, &path_buffer); + var source: safetensors.Tensor = .{ .file_uri = path, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const loader = try Loader.create(failing.allocator(), io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + const parts = [_]LoadPart{.whole(&source)}; + failing.arm(loader); + try expectSubmitError(loader, &.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, error.UnexpectedEndOfFile); +} + +test "source slots keep their own copy of each path" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const contents = [_]u8{ 1, 2, 3, 4 }; + var path_buffer: [1024]u8 = undefined; + const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + // Each submission names the file through a string of its own, freed + // right after the submit: workers open the file through its slot. + for (0..2) |_| { + const uri = try allocator.dupe(u8, path); + var source: safetensors.Tensor = .{ .file_uri = uri, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; + var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; + defer output.deinit(); + const parts = [_]LoadPart{.whole(&source)}; + const submitted = loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none); + allocator.free(uri); + try loader.awaitBatch(try submitted); + try testing.expectShards(allocator, io, &output, &contents, .default); + } + // The testing allocator overwrites freed bytes: a slot keyed by the + // first string would miss the second lookup. + try std.testing.expectEqual(@as(usize, 1), loader.source_slots.count()); +} + +test "column-sharded tensors load as one DMA piece per device and block, read exact or widened" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + const mesh = try devices.mesh("tp"); + + // Rows of 32 bytes, 8 per device; then 12,000 one-byte pieces, more + // than one vectored read takes. + var rows: [64 * 32]u8 = undefined; + var cols: [1024 * 32]u8 = undefined; + var narrow: [3000 * 4]u8 = undefined; + testing.pattern(&rows, 1); + testing.pattern(&cols, 2); + testing.pattern(&narrow, 3); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "staged.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, + .{ .name = "narrow", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &narrow }, + }); + const names = [_][]const u8{ "rows", "cols", "narrow" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + }; + const globals = [_][]const u8{ &rows, &cols, &narrow }; + const row_pieces = 1024 * 4 + 3000 * 4; + + for ([_]usize{ 0, 64 }) |alignment| { + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes) |*part, *spec, *output, name, shape| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .output = output, .parts = part[0..1] }; + } + const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const batch = try loader.submit(&specs, .none); + // Every plan is published once `submit` returns. + var staged_jobs: usize = 0; + var submissions: usize = 0; + for (batch.plans.items) |plan| { + for (plan.jobs) |job| staged_jobs += @intFromBool(job.kind == .dma_land); + submissions += plan.events.len; + } + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); + try std.testing.expect(staged_jobs >= 1); + try std.testing.expect(submissions < row_pieces / 100); + // Every credit is free once the workers stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + } +} + +test "a strided part of a remote file is checked against its exact source range" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 1); + defer devices.deinit(); + var bytes: [128 * 128]u8 = undefined; + testing.pattern(&bytes, 1); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "square.safetensors", ®istry, &.{ + .{ .name = "square", .dtype = "U8", .dims = &.{ 128, 128 }, .bytes = &bytes }, + }); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + // The same file through a registered scheme: a remote file, read exact. + try vfs.registerBackend("remote", .{ .io = io }); + const local = registry.tensors.get("square").?; + var remote = local; + var uri_buffer: [1100]u8 = undefined; + remote.file_uri = try std.fmt.bufPrint(&uri_buffer, "remote://{s}", .{local.file_uri}); + std.debug.assert(local.offset % 4096 != 0); + + var transposed: [128 * 128]u8 = undefined; + for (0..128) |column| { + for (0..128) |row| transposed[column * 128 + row] = bytes[row * 128 + column]; + } + // Transposed: its source read whole by every job, a request exactly, + // at an offset off the direct I/O alignment. + const dims = [_]backend.Lattice.Dim{ + .{ .count = 128, .src_stride = 1, .dst_stride = 128 }, + .{ .count = 128, .src_stride = 128, .dst_stride = 1 }, + }; + const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ .read_parallelism = 2 }); + defer loader.destroy(); + try std.testing.expectEqual(VFS.direct_io_alignment, loader.reads.alignment); + loader.plan_config.request_size = bytes.len; + + var output: Buffer = undefined; + const part: LoadPart = .{ .source = &remote, .lattice = .{ .width = 1, .dims = &dims } }; + const fit = loader.stridedRead(part); + try std.testing.expectEqual(fit.limit, fit.read); + const spec: LoadSpec = .{ .shape = .init(.{ 128, 128 }, .u8), .output = &output, .parts = (&part)[0..1] }; + try loader.awaitBatch(try loader.submit(&.{spec}, .none)); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &transposed, .default); + + // A local file may be read directly: its range widened to the alignment + // no longer fits. + if (VFS.direct_io_alignment != 0) { + const local_part: LoadPart = .{ .source = &local, .lattice = .{ .width = 1, .dims = &dims } }; + const widened = loader.stridedRead(local_part); + try std.testing.expect(widened.read > widened.limit); + } +} + +/// A host-resident load on four devices: rows sharded; columns sharded, two +/// bytes per device per row, so one job has 8,192 transfers and scatters in +/// several reads; replicated; one source twice, replicated and row-sharded, +/// which overlaps; and a device-memory tensor in the same file. +const HostResidentTest = struct { + /// Each run writes its own file from `seed`: a buffer that lands on + /// memory an earlier run freed must not find the expected bytes there. + fn run(devices: *const testing.Devices, seed: u8, alignment: usize) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = devices.platform; + const mesh = try devices.mesh("tp"); + + var rows: [64 * 16]u8 = undefined; + var dev: [16 * 16]u8 = undefined; + var cols: [2048 * 8]u8 = undefined; + var rep: [96 * 4]u8 = undefined; + var twice: [32 * 4]u8 = undefined; + testing.pattern(&rows, seed); + testing.pattern(&dev, seed +% 1); + testing.pattern(&cols, seed +% 2); + testing.pattern(&rep, seed +% 3); + testing.pattern(&twice, seed +% 4); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + // The device tensor separates the host runs: an overlap sends its + // whole job through blocks, so the twice-bound source must not share + // a job with the others. + try testing.writeSafetensors(allocator, io, tmp.dir, "host.safetensors", ®istry, &.{ + .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &rows }, + .{ .name = "cols", .dtype = "U8", .dims = &.{ 2048, 8 }, .bytes = &cols }, + .{ .name = "rep", .dtype = "U8", .dims = &.{ 96, 4 }, .bytes = &rep }, + .{ .name = "dev", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &dev }, + .{ .name = "twice", .dtype = "U8", .dims = &.{ 32, 4 }, .bytes = &twice }, + }); + + const names = [_][]const u8{ "rows", "dev", "cols", "rep", "twice", "twice" }; + const shapes = [_]Shape{ + Shape.init(.{ .row = 64, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), + Shape.init(.{ .row = 2048, .col = 8 }, .u8).withPartitioning(&mesh, .{ .col = .model }), + Shape.init(.{ .row = 96, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{}), + Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{ .row = .model }), + }; + // `rows` as views of pinned memory in the devices' default space. + const memories = [_]Memory{ .host_mapped, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; + const globals = [_][]const u8{ &rows, &dev, &cols, &rep, &twice, &twice }; + var parts: [names.len]LoadPart = undefined; + var specs: [names.len]LoadSpec = undefined; + var outputs: [names.len]Buffer = undefined; + for (&parts, &specs, &outputs, names, shapes, memories) |*part, *spec, *output, name, shape, memory| { + part.* = .whole(registry.tensors.getPtr(name).?); + spec.* = .{ .shape = shape, .memory = memory, .output = output, .parts = part[0..1] }; + } + + const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); + defer loader.destroy(); + loader.reads.alignment = alignment; + const live_before = SharedHostAllocation.live.load(.acquire); + { + const batch = try loader.submit(&specs, .none); + try loader.awaitBatch(batch); + defer for (&outputs) |*output| output.deinit(); + // Host shards in each device's host memory, or its default + // memory for views. + for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); + // Every credit, with its blocks, came back once the workers + // stopped (`stopWorkers`). + loader.stopWorkers(); + try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); + + // The replicated host ones are one allocation. + for (outputs[3..5]) |output| { + const shards = output._shards.constSlice(); + const first = try shards[0].opaqueDeviceMemoryDataPointer(platform.pjrt_api); + for (shards[1..]) |shard| { + const pointer = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api); + try std.testing.expect(pointer == first); + } + } + // The two replicated outputs, and one per device of `rows`. + try std.testing.expectEqual(live_before + 2 + outputs[0]._shards.len, SharedHostAllocation.live.load(.acquire)); + // Shards may be freed in any order: free one replicated output's + // last first; its allocation goes with its last view. + std.mem.reverse(*pjrt.Buffer, outputs[3]._shards.slice()); + } + // PJRT may run the views' deletion callbacks on its own threads, + // just after the shards are freed. + var polls: usize = 0; + while (SharedHostAllocation.live.load(.acquire) != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } + } +}; + +test "host-resident destinations are written in place, through blocks when reads are widened or overlap, and replicas share one allocation" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var devices: testing.Devices = undefined; + try devices.init(allocator, io, 4); + defer devices.deinit(); + if (!backend.supportsMemory(devices.platform, .host_pinned)) return error.SkipZigTest; + // In place, then through blocks for every host job. + for ([_]usize{ 0, 64 }, [_]u8{ 1, 5 }) |alignment, seed| { + try HostResidentTest.run(&devices, seed, alignment); + } +} diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig new file mode 100644 index 00000000..e744a9e6 --- /dev/null +++ b/zml/io/dma_calibration.zig @@ -0,0 +1,675 @@ +//! DMA sizing for the direct backend: the block size, decided once when the +//! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or +//! the one block `zml.io.Loader.Options.dma_block_size` gives, on one +//! representative device, at `depth` transfers per device and over fixed +//! windows, and keeps the smallest block within 8% of the measured peak; +//! borderline candidates get longer, alternating confirmation windows. The +//! ring it measures in stays mapped as the load's first pinned capacity. +//! CPU returns the smallest candidate without measuring: its plugin's +//! transfer is a memcpy, and a load takes the same time at every block +//! size. The buffered backend has nothing to size. +//! +//! `measure` runs the same warm-up and screen without a loader, with every +//! knob, for load_check's `dma-bench` (examples/io/load_check.zig). +const std = @import("std"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Platform = @import("../platform.zig").Platform; +const backend = @import("backend.zig"); +const host_memory = @import("host_memory.zig"); +const log = @import("log.zig").io; + +/// Blocks in flight per device: the screen runs `depth` transfers (the +/// default `Options.block_parallelism`), and the direct backend's DMA +/// budget is `depth` blocks per device, in bytes (`DevicePump.hasRoom` in +/// direct_loader.zig). Eight repeatedly won or tied on MI300X; wider stages +/// mostly increased callback latency and pinned memory, and an adaptive DMA +/// width added state with little load benefit. At 32, four GB300 loaded +/// DeepSeek 9% faster for 2.5x the pinned memory. +pub const depth: usize = 8; + +/// Screened, not fixed: eight MI300X loaded replicated Gemma in 7.83 s at +/// 16 MiB against 10.69 s at 8, whereas one B70 lost 10.5% at 16 MiB. +pub const default_block_sizes = [_]usize{ + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, +}; + +/// The screen's knobs. The loader sets only `block_sizes` and keeps the +/// other defaults; `measure` takes them all. +pub const Options = struct { + block_sizes: []const usize = &default_block_sizes, + /// Transfers in flight per device in the screen, at most + /// `max_block_parallelism`. The loader keeps `depth`. + block_parallelism: usize = depth, + /// A screen window runs for at least this long and, unless the target is + /// zero, until the representative device completes the transfer target. + /// On eight MI300X, 2 ms/32-transfer windows calibrate in 0.96 s against + /// 4.83 s at 10 ms/128, selecting the same block. Short screens sometimes + /// selected the wrong block under noise, hence the longer borderline + /// confirmation windows (`confirmation_duration`). + minimum_duration: std.Io.Duration = .fromMilliseconds(2), + minimum_transfers: u64 = 32, + /// Prefer the smallest block within this tolerance of the measured peak + /// transfer throughput. + block_selection_tolerance: f64 = 0.08, +}; + +/// Borderline block candidates receive longer paired windows, alternating +/// which runs first. A candidate is borderline when its rounds disagree on +/// whether it is near the peak, or when its median ratio to the peak lies +/// within `confirmation_margin` of the tolerance's floor. +const confirmation_duration: std.Io.Duration = .fromMilliseconds(25); +const confirmation_minimum_transfers: u64 = 256; +const confirmation_margin: f64 = 0.02; + +/// The widest screen `measure` takes. +pub const max_block_parallelism: usize = 32; + +/// Measures as the direct loader's creation does, without a loader and with +/// every knob of `opts`, and returns the block it selects: warms the device +/// allocators, then screens the blocks in a workspace of its own, placed +/// near the devices as for reads through the page cache (it reads no file), +/// released before it returns. CPU returns the smallest block without +/// measuring; a platform that loads through the buffered backend +/// (`backend.loadsDirect`) returns `UnsupportedPlatform`, and a knob out of +/// range `InvalidOptions`. Every failure is logged. For load_check's +/// `dma-bench`. +pub fn measure(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) !usize { + try validate(opts); + if (!backend.loadsDirect(platform)) { + log.err("DMA calibration: {t} loads through the buffered backend, which has no DMA to measure: UnsupportedPlatform", .{platform.target}); + return error.UnsupportedPlatform; + } + platform.warmupDeviceAllocators(io) catch |err| { + log.failure(err, "DMA calibration: warm the device allocators: target={t}", .{platform.target}); + return err; + }; + var workspace = try host_memory.Workspace.init(allocator, io, platform, false); + defer workspace.deinit(); + return calibrate(&workspace, platform, opts) catch |err| { + log.failure(err, "DMA calibration: target={t}", .{platform.target}); + return err; + }; +} + +/// A block is at most the largest source request, the bound +/// `zml.io.Loader.init` puts on its own block sizes, so every +/// candidate's ring of `block_parallelism` blocks fits the workspace's +/// mapped ceiling. +fn validate(opts: Options) error{InvalidOptions}!void { + comptime std.debug.assert(backend.max_read_request_size * max_block_parallelism <= host_memory.Workspace.mapped_bytes_ceiling); + if (opts.block_parallelism == 0 or opts.block_parallelism > max_block_parallelism) { + log.err("invalid DMA calibration options: block_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.block_parallelism, max_block_parallelism }); + return error.InvalidOptions; + } + if (opts.block_sizes.len == 0) { + log.err("invalid DMA calibration options: block_sizes is empty: InvalidOptions", .{}); + return error.InvalidOptions; + } + for (opts.block_sizes) |block_size| { + if (block_size == 0 or block_size > backend.max_read_request_size) { + log.err("invalid DMA calibration options: block size {d}, expected 1..{d}: InvalidOptions", .{ block_size, backend.max_read_request_size }); + return error.InvalidOptions; + } + } + if (opts.minimum_duration.nanoseconds <= 0) { + log.err("invalid DMA calibration options: minimum_duration={f}, expected more than zero: InvalidOptions", .{opts.minimum_duration}); + return error.InvalidOptions; + } + // Written so that NaN fails too. + if (!(opts.block_selection_tolerance >= 0 and opts.block_selection_tolerance < 1)) { + log.err("invalid DMA calibration options: block_selection_tolerance={d}, expected [0, 1): InvalidOptions", .{opts.block_selection_tolerance}); + return error.InvalidOptions; + } +} + +/// Measures one representative device into the direct loader's workspace +/// and returns the block it selects; CPU returns the smallest without +/// measuring. On a platform mixing device kinds, device 0's block serves +/// every device: it is a tuning choice, valid for all of them. `opts` were +/// checked (`validate`, or `zml.io.Loader.init` for the loader's block +/// sizes). The rate the `dma_bench` line logs is that one device's +/// synthetic H2D rate, not the platform's nor a prediction of a +/// checkpoint's load speed. +pub fn calibrate( + workspace: *host_memory.Workspace, + platform: *const Platform, + opts: Options, +) !usize { + // Every candidate loads alike on CPU: four PJRT CPU devices loaded warm + // sharded Llama in 1.55-1.60 s at 2, 8 and 16 MiB, so the smallest, + // which maps the least, is taken unmeasured and the loader grows its + // own arenas. + if (platform.target == .cpu) return std.mem.min(usize, opts.block_sizes); + + const allocator = workspace.allocator; + const io = workspace.io; + const start: std.Io.Timestamp = .now(io, .awake); + + var max_total_bytes: usize = 0; + for (opts.block_sizes) |block_size| { + max_total_bytes = @max(max_total_bytes, block_size * opts.block_parallelism); + } + // Map one ring for the largest candidate at the fixed transfer width; + // every candidate reuses it, and the load keeps it. + const calibration_source = try workspace.allocate(max_total_bytes); + + var session = try Session.init(allocator, io, platform, opts.block_sizes, opts.block_parallelism); + defer session.deinit(); + + const candidates = try allocator.alloc(Candidate, opts.block_sizes.len); + defer allocator.free(candidates); + for (candidates, opts.block_sizes, 0..) |*candidate, block_size, index| { + candidate.* = .{ .block_size = block_size, .buffer_offset = index * opts.block_parallelism }; + } + try measureAllCandidates(&session, opts, candidates, calibration_source); + const winner = candidates[try selectCandidate(&session, opts, candidates, calibration_source)]; + + log.debug("dma_bench platform={s} devices={d} kind=\"{s}\" block_bytes={d} parallelism={d} measured_gib_s={d:.3} duration={f} retained_mapped_bytes={d}", .{ + @tagName(platform.target), + platform.devices.len, + platform.devices[0].kind(), + winner.block_size, + opts.block_parallelism, + winner.median().bytesPerSecond() / (1024 * 1024 * 1024), + start.durationTo(.now(io, .awake)), + workspace.mappedBytes(), + }); + return winner.block_size; +} + +const sample_count = 3; + +fn measureAllCandidates( + session: *Session, + opts: Options, + candidates: []Candidate, + source: []const u8, +) !void { + for (0..sample_count) |round| { + for (0..candidates.len) |i| { + const candidate = &candidates[(i + round) % candidates.len]; + candidate.metrics[round] = try runWindow( + session.io, + session, + candidate, + source, + opts.minimum_duration, + opts.minimum_transfers, + ); + } + } +} + +/// Returns the index of the selected candidate (`pick`), over each +/// candidate's median ratio to the peak, or its confirmation's when it is +/// borderline. +fn selectCandidate( + session: *Session, + opts: Options, + candidates: []Candidate, + source: []const u8, +) !usize { + const ratios = try session.allocator.alloc(f64, candidates.len); + defer session.allocator.free(ratios); + + var peak_index: usize = 0; + var peak_rate = candidates[0].median().bytesPerSecond(); + for (candidates[1..], 1..) |candidate, index| { + const rate = candidate.median().bytesPerSecond(); + if (rate > peak_rate) { + peak_index = index; + peak_rate = rate; + } + } + for (candidates, ratios) |candidate, *ratio| { + ratio.* = if (peak_rate == 0) 0 else candidate.median().bytesPerSecond() / peak_rate; + } + + // A borderline candidate's ratio is its confirmation's against the peak. + for (candidates, ratios, 0..) |*candidate, *ratio, index| { + if (index != peak_index and isBorderline(candidates, index, ratio.*, opts.block_selection_tolerance)) + ratio.* = try confirmCandidateAgainstPeak(session, candidate, &candidates[peak_index], source); + } + return pick(candidates, ratios, opts.block_selection_tolerance); +} + +/// The smallest block whose ratio is within `tolerance` of the best ratio, +/// screened or confirmed. A confirmation can rate a block above the screen's +/// peak, so the best ratio may belong to a larger block. +fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usize { + var best_index: usize = 0; + for (ratios, 0..) |ratio, index| { + if (ratio > ratios[best_index]) best_index = index; + } + const floor = ratios[best_index] * (1.0 - tolerance); + var selected_index = best_index; + for (candidates, ratios, 0..) |candidate, ratio, index| { + if (ratio >= floor and candidate.block_size < candidates[selected_index].block_size) + selected_index = index; + } + return selected_index; +} + +/// Whether `candidates[index]`, at screen `ratio` to the peak, gets +/// confirmation windows (`confirmation_margin`). +fn isBorderline(candidates: []const Candidate, index: usize, ratio: f64, tolerance: f64) bool { + // Whether the candidate is near-peak in one round but fails in others. + var qualified_once = false; + var rejected_once = false; + for (candidates[index].metrics, 0..) |metric, round| { + var round_peak_rate: f64 = 0; + for (candidates) |round_candidate| { + round_peak_rate = @max(round_peak_rate, round_candidate.metrics[round].bytesPerSecond()); + } + const round_ratio = if (round_peak_rate == 0) 0 else metric.bytesPerSecond() / round_peak_rate; + if (round_ratio >= 1.0 - tolerance) + qualified_once = true + else + rejected_once = true; + } + return (qualified_once and rejected_once) or @abs(ratio - (1.0 - tolerance)) <= confirmation_margin; +} + +/// The median, over `sample_count` paired windows, of `candidate`'s rate +/// over `peak`'s. +fn confirmCandidateAgainstPeak( + session: *Session, + candidate: *Candidate, + peak: *Candidate, + source: []const u8, +) !f64 { + var ratios: [sample_count]f64 = undefined; + for (&ratios, 0..) |*ratio, repeat| { + const order = if (repeat % 2 == 0) + [_]*Candidate{ candidate, peak } + else + [_]*Candidate{ peak, candidate }; + var candidate_rate: f64 = undefined; + var peak_rate: f64 = undefined; + for (order) |measured| { + const rate = (try runWindow( + session.io, + session, + measured, + source, + confirmation_duration, + confirmation_minimum_transfers, + )).bytesPerSecond(); + if (measured == candidate) candidate_rate = rate else peak_rate = rate; + } + ratio.* = if (peak_rate == 0) 0 else candidate_rate / peak_rate; + } + std.mem.sort(f64, &ratios, {}, std.sort.asc(f64)); + return ratios[ratios.len / 2]; +} + +/// Runs one window of `candidate`'s transfers, `session.parallelism` at a +/// time. `io` sleeps and joins the window; the workers transfer on the +/// session's. +fn runWindow( + io: std.Io, + session: *Session, + candidate: *Candidate, + source: []const u8, + minimum_duration: std.Io.Duration, + minimum_transfers: u64, +) !Measurement { + const Worker = struct { + session: *Session, + buffer_index: usize, + block: []const u8, + metrics: *Counters, + ready: *std.atomic.Value(usize), + start: *std.Io.Event, + stop: *std.atomic.Value(bool), + + fn run(self: @This()) void { + _ = self.ready.fetchAdd(1, .release); + self.start.waitUncancelable(self.session.io); + while (!self.stop.load(.acquire)) { + self.session.transfer(self.buffer_index, self.block, self.metrics); + if (self.session.first_error.get() != null) return; + } + } + }; + + var metrics: Counters = .{}; + var ready: std.atomic.Value(usize) = .init(0); + var start: std.Io.Event = .unset; + var stop: std.atomic.Value(bool) = .init(false); + var group: std.Io.Group = .init; + defer { + // Workers borrow this window's stack. Every exit, including a canceled + // sleep or partial spawn, must release the start gate and join them. + stop.store(true, .release); + start.set(io); + group.await(io) catch {}; + } + for (0..session.parallelism) |slot| { + try group.concurrent(io, Worker.run, .{Worker{ + .session = session, + .buffer_index = candidate.buffer_offset + slot, + .block = candidate.block(source, slot), + .metrics = &metrics, + .ready = &ready, + .start = &start, + .stop = &stop, + }}); + } + + try session.warmup(candidate, source); + + while (ready.load(.acquire) != session.parallelism) try io.sleep(.fromMicroseconds(100), .awake); + const start_at: std.Io.Timestamp = .now(io, .awake); + start.set(io); + while (true) { + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + // is window complete + if (elapsed.nanoseconds >= minimum_duration.nanoseconds and metrics.transfers.load(.acquire) >= minimum_transfers) break; + try io.sleep(.fromMilliseconds(1), .awake); + } + stop.store(true, .release); + try group.await(io); + const elapsed = start_at.durationTo(.now(io, .awake)); + if (session.first_error.get()) |err| return err; + return .{ .bytes = metrics.bytes.load(.acquire), .elapsed = elapsed }; +} + +/// One block size and its screen measurements. Its slots are the session's +/// buffers from `buffer_offset` on, one per transfer in flight. +const Candidate = struct { + block_size: usize, + buffer_offset: usize, + /// Slots warmed so far (`Session.warmup`). + warmed_buffers: usize = 0, + + /// One window per screen round (`measureAllCandidates`). + metrics: [sample_count]Measurement = undefined, + + fn median(self: Candidate) Measurement { + var sorted = self.metrics; + std.mem.sort(Measurement, &sorted, {}, struct { + fn lessThan(_: void, lhs: Measurement, rhs: Measurement) bool { + return lhs.bytesPerSecond() < rhs.bytesPerSecond(); + } + }.lessThan); + return sorted[sorted.len / 2]; + } + + /// The ring's bytes that `slot` transfers. + fn block(self: Candidate, source: []const u8, slot: usize) []const u8 { + return source[slot * self.block_size ..][0..self.block_size]; + } +}; + +/// One transfer manager for the whole screen, sliced per candidate, and the +/// screen's first transfer error, which ends it. +const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + manager: *pjrt.AsyncHostToDeviceTransferManager, + buffers: []*pjrt.Buffer, + /// Each candidate's slots, and the transfers a window runs at a time. + parallelism: usize, + first_error: stdx.FirstError(pjrt.ApiError) = .{}, + + fn transfer(self: *Session, buffer_index: usize, block: []const u8, metrics: ?*Counters) void { + const api = self.platform.pjrt_api; + const event = self.manager.transferData(api, buffer_index, block, 0, false) catch |err| { + _ = self.first_error.set(err); + return; + }; + defer event.deinit(api); + event.await(api, self.io) catch |err| { + _ = self.first_error.set(err); + return; + }; + if (metrics) |output| { + _ = output.bytes.fetchAdd(@intCast(block.len), .monotonic); + _ = output.transfers.fetchAdd(1, .monotonic); + } + } + + /// Transfers twice into each of `candidate`'s slots not yet warmed. + fn warmup(self: *Session, candidate: *Candidate, source: []const u8) !void { + while (candidate.warmed_buffers < self.parallelism) : (candidate.warmed_buffers += 1) { + const slot = candidate.warmed_buffers; + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); + if (self.first_error.get()) |err| return err; + } + } + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + block_sizes: []const usize, + parallelism: usize, + ) !Session { + const buffer_count = std.math.mul(usize, block_sizes.len, parallelism) catch return error.OutOfMemory; + const dims = try allocator.alloc([1]i64, block_sizes.len); + defer allocator.free(dims); + const specs = try allocator.alloc(pjrt.ShapeSpec, buffer_count); + defer allocator.free(specs); + for (block_sizes, dims, 0..) |block_size, *dim, index| { + dim.* = .{@intCast(block_size)}; + @memset(specs[index * parallelism ..][0..parallelism], pjrt.ShapeSpec.init(dim, .u8)); + } + const buffers = try allocator.alloc(*pjrt.Buffer, buffer_count); + errdefer allocator.free(buffers); + const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = specs, + .memory = platform.devices[0].memory(.default).?.pjrt_memory, + }); + var retrieved: usize = 0; + errdefer { + manager.deinit(platform.pjrt_api); + for (buffers[0..retrieved]) |buffer| buffer.deinit(platform.pjrt_api); + } + for (buffers, 0..) |*buffer, index| { + buffer.* = try manager.retrieveBuffer(platform.pjrt_api, index); + retrieved += 1; + } + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .manager = manager, + .buffers = buffers, + .parallelism = parallelism, + }; + } + + fn deinit(self: *Session) void { + // XLA commit 1b19ae012aa67426658f7ca3c1503bb781c863a9 switched GPU + // transfers to CommonAsyncHostToDeviceTransferManager, whose destructor + // waits for outstanding transfers and marks unfinished buffers as errored. + // We discard these buffers, so no final is_last_transfer=true copy is needed. + self.manager.deinit(self.platform.pjrt_api); + for (self.buffers) |buffer| buffer.deinit(self.platform.pjrt_api); + self.allocator.free(self.buffers); + self.* = undefined; + } +}; + +const Counters = struct { + bytes: std.atomic.Value(u64) = .init(0), + transfers: std.atomic.Value(u64) = .init(0), +}; + +const Measurement = struct { + bytes: u64, + elapsed: std.Io.Duration, + + fn bytesPerSecond(self: Measurement) f64 { + if (self.elapsed.nanoseconds <= 0) return 0; + return @as(f64, @floatFromInt(self.bytes)) * std.time.ns_per_s / + @as(f64, @floatFromInt(self.elapsed.nanoseconds)); + } +}; + +test "DMA benchmark selection uses medians and prefers the smallest near-peak value" { + // No candidate below is borderline, so the selector never schedules a + // confirmation window and the session supplies only its allocator. + var session: Session = .{ + .allocator = std.testing.allocator, + .io = undefined, + .platform = undefined, + .manager = undefined, + .buffers = undefined, + .parallelism = undefined, + }; + const opts: Options = .{ .block_selection_tolerance = 0.05 }; + + var candidates = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 60, 10, 62 }, + .{ 98, 99, 97 }, + .{ 100, 101, 99 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + const winner = candidates[try selectCandidate(&session, opts, &candidates, &.{})]; + try std.testing.expectEqual(@as(usize, 4), winner.block_size); + try std.testing.expectEqual(@as(f64, 98), winner.median().bytesPerSecond()); + + // A larger block near the peak in every round is neither confirmed nor + // picked. + var bimodal = [_]Candidate{ + .{ .block_size = 2, .buffer_offset = undefined }, + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + .{ .block_size = 16, .buffer_offset = undefined }, + }; + const bimodal_rates = [_]u64{ 80, 100, 70, 99 }; + for (&bimodal, bimodal_rates) |*candidate, rate| { + candidate.metrics = @splat(.{ .bytes = rate, .elapsed = .fromSeconds(1) }); + } + try std.testing.expectEqual(@as(usize, 4), bimodal[try selectCandidate(&session, opts, &bimodal, &.{})].block_size); +} + +test "DMA benchmark picks the smallest block within the tolerance of the best ratio" { + const candidates = [_]Candidate{ + .{ .block_size = 16 * 1024 * 1024, .buffer_offset = undefined }, + .{ .block_size = 32 * 1024 * 1024, .buffer_offset = undefined }, + }; + // 32 MiB is the peak; 16 MiB within the tolerance below it, then past it. + try std.testing.expectEqual(0, pick(&candidates, &.{ 0.93, 1.0 }, 0.08)); + try std.testing.expectEqual(1, pick(&candidates, &.{ 0.91, 1.0 }, 0.08)); + // 16 MiB is the screen's peak; a confirmation rates 32 MiB more than the + // tolerance above it, then within it. + try std.testing.expectEqual(1, pick(&candidates, &.{ 1.0, 1.10 }, 0.08)); + try std.testing.expectEqual(0, pick(&candidates, &.{ 1.0, 1.05 }, 0.08)); +} + +test "DMA benchmark confirms a candidate when round qualification disagrees" { + var candidates = [_]Candidate{ + .{ .block_size = 4, .buffer_offset = undefined }, + .{ .block_size = 8, .buffer_offset = undefined }, + }; + const rates = [_][sample_count]u64{ + .{ 99, 80, 70 }, + .{ 100, 100, 100 }, + }; + for (&candidates, rates) |*candidate, candidate_rates| { + for (&candidate.metrics, candidate_rates) |*metric, rate| { + metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; + } + } + // The median ratio, 0.80, is far from the 0.95 floor: only the rounds + // make it borderline. + try std.testing.expect(isBorderline(&candidates, 0, 0.80, 0.05)); +} + +test "measure refuses a knob out of range before it touches the platform" { + const cases = [_]Options{ + .{ .block_sizes = &.{} }, + .{ .block_sizes = &.{0} }, + .{ .block_sizes = &.{backend.max_read_request_size + 1} }, + .{ .block_parallelism = 0 }, + .{ .block_parallelism = max_block_parallelism + 1 }, + .{ .minimum_duration = .zero }, + .{ .minimum_duration = .fromMilliseconds(-1) }, + .{ .block_selection_tolerance = 1 }, + .{ .block_selection_tolerance = -0.01 }, + .{ .block_selection_tolerance = std.math.nan(f64) }, + }; + for (cases) |opts| { + try std.testing.expectError(error.InvalidOptions, measure(std.testing.allocator, std.testing.io, undefined, opts)); + } +} + +test "measure needs no loader and selects the smallest block on CPU" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + try std.testing.expectEqual(4 << 20, try measure(allocator, io, platform, .{ .block_sizes = &.{ 8 << 20, 4 << 20 } })); +} + +test "DMA benchmark shares one manager across candidate sizes and slots" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var session = try Session.init(allocator, io, platform, &.{ 8, 16 }, 2); + defer session.deinit(); + const source: [32]u8 = @splat(0); + for ([_]usize{ 8, 16 }, 0..) |block_size, index| { + var candidate: Candidate = .{ .block_size = block_size, .buffer_offset = index * 2 }; + try session.warmup(&candidate, &source); + try std.testing.expectEqual(2, candidate.warmed_buffers); + } + try std.testing.expectEqual(null, session.first_error.get()); +} + +test "DMA benchmark cancellation drains transfer workers" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch + return error.SkipZigTest; + defer platform.deinit(allocator, io); + + var source: [16]u8 = @splat(0); + var session = try Session.init(allocator, io, platform, &.{source.len}, 1); + defer session.deinit(); + var candidate: Candidate = .{ .block_size = source.len, .buffer_offset = 0 }; + var vtable = io.vtable.*; + vtable.sleep = struct { + fn sleep(_: ?*anyopaque, _: std.Io.Timeout) std.Io.Cancelable!void { + return error.Canceled; + } + }.sleep; + const canceled_io: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + // A cancellation can arrive while workers wait for the start gate or + // while they transfer. In either case they must finish before teardown. + try std.testing.expectError(error.Canceled, runWindow( + canceled_io, + &session, + &candidate, + &source, + .max, + 0, + )); +} diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig new file mode 100644 index 00000000..45063b7e --- /dev/null +++ b/zml/io/host_memory.zig @@ -0,0 +1,613 @@ +//! Loader-owned host arenas, cut into blocks once. The direct backend owns +//! its workspace for its whole life: calibration maps its ring there, the +//! backend grows the set to what its load holds and carves it, and its +//! `destroy` releases every arena. Callers never supply a workspace, a +//! budget, a NUMA policy or a calibration result. +//! +//! The memory depends on the target: CUDA and oneAPI take page-backed arenas +//! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` +//! buffers, and CPU plain pages, unregistered. The arenas' kind and the +//! transfer path are separate decisions: CPU loads through the direct +//! backend too. When the task's cpuset and memory policy allow several +//! memory-bearing NUMA nodes, page-backed arenas interleave over those of the +//! platform's devices (all of them when the load reads past the page cache, +//! a device's node is unknown, or none of theirs is allowed); ROCm balances +//! whole arenas across its devices' host nodes. +//! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a +//! target. A failed allocation or registration, or a growth past the +//! ceiling, is logged with its cause and returns `OutOfMemory`. + +const std = @import("std"); +const Alignment = std.mem.Alignment; +const Allocator = std.mem.Allocator; +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); + +const Platform = @import("../platform.zig").Platform; +const log = @import("log.zig").mem; + +/// One ROCm host-memory allocation path and its bytes allocated so far. +const HostNode = struct { + /// The plugin's `numa_node` device attribute (`Device.numaNode`); null + /// for the one path kept when no device reports it. + node: ?u32, + any_device_index: usize, + total_allocated_bytes: usize = 0, +}; + +/// One retained arena. The bytes of a `pinned_host` arena belong to its +/// PJRT buffer; page-backed arenas have none. +const Arena = struct { + bytes: []u8, + pjrt_buffer: ?*pjrt.Buffer = null, +}; + +/// How arenas are mapped and unmapped; each strategy keeps only the state +/// it uses. +const ArenaKind = union(enum) { + /// With eight MI300X visible, hipHostRegister took ~6.3 s for 1 GiB + /// versus 0.6-0.7 s through hipHostMalloc. KFD/IOMMU registration maps + /// pages to every GPU; huge-page advice helped only ~9%, and touching + /// pages or changing HIP flags did not remove the cost. Standard PJRT + /// pinned_host buffers provide allocation-owned memory without a custom + /// allocator extension. + pjrt_host: struct { + platform: *const Platform, + host_nodes: std.ArrayList(HostNode), + }, + /// Page-backed arenas, registered with the platform's PJRT client when + /// there is one (`dma_map` in the arena log) and left pageable for CPU + /// transfers when not (`pageable`), with huge-page alignment and advice. + /// Registration only avoids staging if the plugin recognizes the range as + /// pinned, including subranges. The oneAPI plugin must implement + /// IsHostMemoryPinned for SYCL imports: without it, one B70 staged + /// transfers at 11.01 GiB/s instead of 26.90 (eight DMA caps). + pages: struct { + platform: ?*const Platform, + /// Zero leaves placement to the kernel. Automatic placement can + /// fall back to zero if the kernel refuses it. + numa_mask: u64, + }, + + const transparent_huge_page_size = 2 * 1024 * 1024; + const mpol_interleave = 3; + + fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + direct_reads: bool, + ) Allocator.Error!ArenaKind { + return switch (platform.target) { + .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, + .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, + // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory + // from `/dev/dma_heap/system`, which the SDK recognizes by + // address and transfers without pinning pages on every request. + .rocm, .furiosa => { + // PJRT-pinned allocations choose placement through the device's + // host memory space, not our mbind policy. Retain one allocation + // path per reported node; the blocks stay unpartitioned. + var host_nodes: std.ArrayList(HostNode) = try .initCapacity(allocator, @max(platform.devices.len, 1)); + devices: for (platform.devices, 0..) |device, device_index| { + std.debug.assert(device.memory(.host_pinned) != null); + const node = device.numaNode() orelse continue; + for (host_nodes.items) |existing| { + if (existing.node == node) continue :devices; + } + host_nodes.appendAssumeCapacity(.{ .node = node, .any_device_index = device_index }); + } + if (host_nodes.items.len == 0) host_nodes.appendAssumeCapacity(.{ .node = null, .any_device_index = 0 }); + return .{ .pjrt_host = .{ .platform = platform, .host_nodes = host_nodes } }; + }, + .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), + }; + } + + /// Call after every arena is unmapped. + fn deinit(self: *ArenaKind, allocator: Allocator) void { + switch (self.*) { + .pjrt_host => |*host| host.host_nodes.deinit(allocator), + .pages => {}, + } + } + + /// Maps one arena of `bytes`. + fn map(self: *ArenaKind, allocator: Allocator, io: std.Io, bytes: usize) Allocator.Error!Arena { + const started: std.Io.Timestamp = .now(io, .awake); + switch (self.*) { + .pjrt_host => |*host| { + // Balance bytes, not arena/device counts: arenas differ in size. + // Replicated Llama-3.1-8B on eight MI300X lost the fast mode when + // all arenas came from device zero (1.27-1.45 s). Device rotation + // left a 61/39 byte split and took 0.98-1.33 s; byte-balanced + // allocation recovered 0.84-1.05 s over six runs, with 1.52 GiB + // mapped versus 2.03 GiB for per-node pools. Host contention + // affected both arms. + var host_node = &host.host_nodes.items[0]; + for (host.host_nodes.items[1..]) |*candidate| { + if (candidate.total_allocated_bytes < host_node.total_allocated_bytes) host_node = candidate; + } + const device_index = host_node.any_device_index; + // PJRT is the sole owner of these hipHostMalloc-backed bytes. + // Never DmaMap or DmaUnmap the borrowed pointer: unregistering + // allocation-owned memory breaks the eventual hipHostFree. The + // buffer and its external reference both live until `unmap`, + // after all reads and DMA finish. + // The ROCm plugin must implement IsHostMemoryPinned: without + // it, one MI300X staged transfers at 6.5 GiB/s instead of 46.6. + const api = host.platform.pjrt_api; + const memory = host.platform.devices[device_index].memory(.host_pinned).?; + const buffer = host.platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = &.{@intCast(bytes)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + log.err("pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.deinit(api); + std.debug.assert(buffer.isOnCpu(api)); + buffer.increaseExternalReferenceCount(api) catch |err| { + log.err("retaining pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }; + errdefer buffer.decreaseExternalReferenceCount(api) catch {}; + const ptr: [*]u8 = @ptrCast(buffer.opaqueDeviceMemoryDataPointer(api) catch |err| { + log.err("accessing pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); + return error.OutOfMemory; + }); + // Counted once mapped: the workspace retains every mapped arena. + host_node.total_allocated_bytes += bytes; + log.debug("DMA arena kind=pjrt_host device={d} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + device_index, + @intFromPtr(ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, .now(io, .awake)))) / std.time.ns_per_ms, + }); + return .{ .bytes = ptr[0..bytes], .pjrt_buffer = buffer }; + }, + .pages => |*pages| { + const alignment = pageAlignment(bytes); + const ptr = allocator.rawAlloc(bytes, alignment, @returnAddress()) orelse { + log.err("host memory allocation failed for {Bi:.2}: OutOfMemory", .{bytes}); + return error.OutOfMemory; + }; + const data = ptr[0..bytes]; + place(&pages.numa_mask, data); + adviseHugePages(data); + if (pages.platform) |p| p.pjrt_client.dmaMap(p.pjrt_api, data) catch |err| { + log.err("DMA registration failed for {Bi:.2} of host memory: {s}", .{ bytes, @errorName(err) }); + allocator.rawFree(data, alignment, @returnAddress()); + return error.OutOfMemory; + }; + const finished: std.Io.Timestamp = .now(io, .awake); + log.debug("DMA arena kind={s} placement={s} nodes=0x{x} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ + if (pages.platform != null) "dma_map" else "pageable", + if (pages.numa_mask == 0) "unplaced" else "interleave", + pages.numa_mask, + @intFromPtr(data.ptr), + bytes, + @as(f64, @floatFromInt(elapsedNanoseconds(started, finished))) / std.time.ns_per_ms, + }); + return .{ .bytes = data }; + }, + } + } + + fn unmap(self: *const ArenaKind, allocator: Allocator, arena: Arena) void { + switch (self.*) { + .pjrt_host => |host| { + const api = host.platform.pjrt_api; + const buffer = arena.pjrt_buffer.?; + buffer.decreaseExternalReferenceCount(api) catch unreachable; + buffer.deinit(api); + }, + .pages => |pages| { + if (pages.platform) |platform| { + // Unregistering can fail after a device error. The pages may + // still be registered, so they are leaked, not reused. + platform.pjrt_client.dmaUnmap(platform.pjrt_api, arena.bytes) catch |err| { + log.warn("DMA unregistration failed for {Bi:.2} of host memory, which stays allocated: {s}", .{ arena.bytes.len, @errorName(err) }); + return; + }; + } + allocator.rawFree(arena.bytes, pageAlignment(arena.bytes.len), @returnAddress()); + }, + } + } + + /// Page alignment, raised to the huge-page size for arenas that span one. + fn pageAlignment(len: usize) Alignment { + const alignment: Alignment = comptime .fromByteUnits(std.heap.page_size_min); + if (comptime builtin.os.tag != .linux) return alignment; + if (len < transparent_huge_page_size) return alignment; + return alignment.max(.fromByteUnits(transparent_huge_page_size)); + } + + /// Interleaves `data` over `numa_mask`'s nodes; on refusal, warns and + /// zeroes the mask, so later arenas stay unplaced too. + fn place(numa_mask: *u64, data: []u8) void { + const mask = numa_mask.*; + if (mask == 0) return; + if (comptime builtin.os.tag != .linux) return; + + const node_mask: [1]u64 = .{mask}; + const highest_node: usize = 63 - @clz(mask); + const rc = std.os.linux.syscall6( + .mbind, + @intFromPtr(data.ptr), + data.len, + mpol_interleave, + @intFromPtr(&node_mask), + // Linux get_nodes() decrements maxnode before copying the mask; + // raw callers include the same extra sentinel bit as libnuma. + highest_node + 2, + 0, + ); + if (std.os.linux.errno(rc) == .SUCCESS) return; + log.warn("NUMA placement of DMA arenas over nodes 0x{x} refused ({s}); leaving them unplaced", .{ + mask, + @tagName(std.os.linux.errno(rc)), + }); + // Automatic placement is not retried: the arenas stay unplaced. + numa_mask.* = 0; + } + + fn adviseHugePages(data: []u8) void { + if (comptime builtin.os.tag != .linux) return; + if (data.len < transparent_huge_page_size) return; + + const ptr: [*]align(std.heap.page_size_min) u8 = @alignCast(data.ptr); + std.posix.madvise(ptr, data.len, std.posix.MADV.HUGEPAGE) catch |err| { + log.warn("MADV_HUGEPAGE failed for DMA buffer at 0x{x} ({Bi:.2}): {s}", .{ + @intFromPtr(data.ptr), + data.len, + @errorName(err), + }); + }; + } +}; + +/// Host arenas owned by one direct loader. Calibration maps its ring +/// (`allocate`); the loader then grows the set to what its load holds and +/// cuts it into blocks, once (`carve`). Deinitialize after all transfers +/// finish and before the platform. +pub const Workspace = struct { + /// Safety guard on the arenas' total host memory (pinned on the DMA + /// targets), not an allocation target: calibration and the load's + /// pre-growth decide what is actually mapped. Fixed, because no caller + /// ever had a reason to choose another value. + pub const mapped_bytes_ceiling: usize = 16 * 1024 * 1024 * 1024; + + allocator: Allocator, + io: std.Io, + kind: ArenaKind, + /// Every arena mapped, in mapping order, until `deinit`. + arenas: std.ArrayList(Arena) = .empty, + /// The arenas cut into blocks, set by `carve`: nothing maps after it. + blocks: ?[]Block = null, + + /// `direct_reads`: the load reads its checkpoint past the page cache, so + /// page-backed arenas spread over every candidate node + /// (`placementMask`). + pub fn init( + allocator: Allocator, + io: std.Io, + platform: *const Platform, + direct_reads: bool, + ) Allocator.Error!Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = try .init(allocator, io, platform, direct_reads), + }; + } + + pub fn deinit(self: *Workspace) void { + const io = self.io; + const mapped_bytes = self.mappedBytes(); + const started: std.Io.Timestamp = .now(io, .awake); + if (self.blocks) |blocks| self.allocator.free(blocks); + for (self.arenas.items) |arena| self.kind.unmap(self.allocator, arena); + self.arenas.deinit(self.allocator); + self.kind.deinit(self.allocator); + const elapsed_ns = elapsedNanoseconds(started, .now(io, .awake)); + log.debug("DMA load workspace teardown: mapped={Bi:.2}, elapsed_ms={d:.3}", .{ + mapped_bytes, + @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms, + }); + self.* = undefined; + } + + /// The bytes of every retained arena, tails included. + pub fn mappedBytes(self: *const Workspace) usize { + var mapped: usize = 0; + for (self.arenas.items) |arena| mapped += arena.bytes.len; + return mapped; + } + + /// Retains one new arena within the mapped-byte ceiling, before `carve`. + pub fn allocate(self: *Workspace, bytes: usize) Allocator.Error![]u8 { + std.debug.assert(self.blocks == null); + const mapped = self.mappedBytes(); + if (bytes > mapped_bytes_ceiling - mapped) { + log.err("host memory allocation exceeds mapped ceiling: requested={Bi:.2}, mapped={Bi:.2}, ceiling={Bi:.2}: OutOfMemory", .{ bytes, mapped, mapped_bytes_ceiling }); + return error.OutOfMemory; + } + // Reserved first, so a mapped arena is always retained: the ROCm + // balancer counts the bytes it maps. + try self.arenas.ensureUnusedCapacity(self.allocator, 1); + const arena = try self.kind.map(self.allocator, self.io, bytes); + self.arenas.appendAssumeCapacity(arena); + return arena.bytes; + } + + /// The blocks of `block_size` the workspace can hold: those of its + /// retained arenas, and those the mapped ceiling still allows. + pub fn capacity(self: *const Workspace, block_size: usize) usize { + return self.usableBlocks(block_size) + (mapped_bytes_ceiling - self.mappedBytes()) / block_size; + } + + /// Maps the blocks missing below `first_blocks` as one arena, then those + /// missing below `total_blocks` as another, and cuts every retained arena + /// into blocks of `block_size`, once. The set is then complete: nothing + /// maps inside a load (a slab mapped there cost 146 to 230 ms of + /// hipHostMalloc on MI300X), and `allocate` asserts it. Arena tails + /// smaller than one block stay mapped and unused. Blocks are not split + /// per NUMA node: strict per-node pools duplicated source reserves and + /// constrained reads to the smallest node; placement belongs to arena + /// allocation. The workspace owns the list and the memory. + pub fn carve(self: *Workspace, block_size: usize, first_blocks: usize, total_blocks: usize) Allocator.Error![]const Block { + std.debug.assert(self.blocks == null and block_size != 0); + for ([_]usize{ first_blocks, total_blocks }) |target_blocks| { + const missing_blocks = target_blocks -| self.usableBlocks(block_size); + if (missing_blocks != 0) _ = try self.allocate(missing_blocks * block_size); + } + const blocks = try self.allocator.alloc(Block, self.usableBlocks(block_size)); + var next: usize = 0; + for (self.arenas.items) |arena| { + for (0..arena.bytes.len / block_size) |j| { + blocks[next] = arena.bytes[j * block_size ..][0..block_size]; + next += 1; + } + } + std.debug.assert(next == blocks.len); + self.blocks = blocks; + return blocks; + } + + /// Counts complete blocks across retained arenas; block_size must be nonzero. + fn usableBlocks(self: *const Workspace, block_size: usize) usize { + var usable: usize = 0; + for (self.arenas.items) |arena| usable += arena.bytes.len / block_size; + return usable; + } + + /// Creates ordinary allocator-backed arenas for tests without a PJRT platform. + fn initForTesting(allocator: Allocator, io: std.Io) Workspace { + return .{ + .allocator = allocator, + .io = io, + .kind = .{ .pages = .{ .platform = null, .numa_mask = 0 } }, + }; + } +}; + +pub const Block = []u8; + +/// Bits of `/sys/devices/system/node/has_memory`, or zero when unreadable. +/// Nodes 64 and above cannot be represented and are dropped. +/// Device-coherent HBM nodes are not candidates for host pages: the measured +/// four-GB300 topology had 34 NUMA nodes but only two with host memory. +fn memoryNodeMask(allocator: Allocator, io: std.Io) u64 { + if (comptime builtin.os.tag != .linux) return 0; + const contents = std.Io.Dir.cwd().readFileAlloc( + io, + "/sys/devices/system/node/has_memory", + allocator, + .limited(4096), + ) catch return 0; + defer allocator.free(contents); + return parseNodeList(contents); +} + +/// The nodes the task's memory policy allows (`numactl --membind`, +/// `--interleave`, `--preferred`); every node under the default policy. +/// An arena placed outside them would defeat the policy: `mbind` on a +/// range overrides `set_mempolicy`. +fn taskMemoryPolicyMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + var mode: c_int = 0; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, @intFromPtr(&mode), @intFromPtr(&nodes), 64, 0, 0); + if (std.os.linux.errno(rc) != .SUCCESS or mode == 0 or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes the task's cpuset lets it allocate on (`Mems_allowed`), or +/// every node when unknown. `mbind` refuses a mask with none of them. +fn cpusetMemsMask() u64 { + if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); + const mpol_f_mems_allowed = 4; + var nodes: [1]u64 = .{0}; + const rc = std.os.linux.syscall5(.get_mempolicy, 0, @intFromPtr(&nodes), 64, 0, mpol_f_mems_allowed); + if (std.os.linux.errno(rc) != .SUCCESS or nodes[0] == 0) return std.math.maxInt(u64); + return nodes[0]; +} + +/// The nodes page-backed arenas interleave over (`placementMask`): the +/// memory-bearing nodes the task's cpuset and memory policy allow, narrowed +/// to the nodes of the platform's devices, unless the load reads past the +/// page cache. +fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform, direct_reads: bool) u64 { + const candidates = memoryNodeMask(allocator, io) & cpusetMemsMask() & taskMemoryPolicyMask(); + return placementMask(candidates, deviceNodeMask(platform), direct_reads); +} + +/// The nodes of the platform's devices (`Device.numaNode`), or null when a +/// device reports none (the CPU plugin never does) or a node past 63. +fn deviceNodeMask(platform: *const Platform) ?u64 { + var mask: u64 = 0; + for (platform.devices) |device| { + const node = device.numaNode() orelse return null; + if (node >= 64) return null; + mask |= @as(u64, 1) << @intCast(node); + } + return if (mask == 0) null else mask; +} + +/// Zero leaves placement to the kernel: with one candidate node or none +/// there is nothing to choose, and the task's policy or the only node decides. +/// For reads through the page cache, the arenas go to the devices' nodes +/// among several candidates: an arena on another node crosses the +/// inter-socket link on every H2D copy. Two GB300 on one Grace node loaded +/// Qwen3.8-27B-FP8 in 1.265 s with a task policy holding the arenas to that +/// node, against 1.64 s interleaved over both nodes. Devices on several +/// nodes share one pool interleaved over theirs: four GB300 on two nodes +/// loaded replicated DeepSeek-V4-Flash in 4.35 s against 4.92 s with +/// per-node pools. When a device's node is unknown, or none is a candidate, +/// the arenas interleave over every candidate, which no single node beat +/// everywhere. A single node is placed too, since unplaced CUDA arenas +/// follow the registering thread's node; the kernel still falls back when +/// it is full. +/// A load read past the page cache (`direct_reads`) spreads the arenas over +/// every candidate instead: the drives write into them, and interleaving +/// splits that traffic, and the copies out of them, between the candidates' +/// memory. Over an md RAID0 of four NVMe drives, two on each node, the same +/// two GB300 loaded the same checkpoint forced direct in 1.752 s with the +/// arenas interleaved over both nodes, against 1.873 s with them on the +/// devices' node: staging a read took 0.52 against 0.64 ms, and a 16 MiB +/// read 1.90 against 1.87 ms. The drives' own nodes are not consulted: every +/// candidate matched them on the one host measured. +fn placementMask(candidates: u64, device_nodes: ?u64, direct_reads: bool) u64 { + if (@popCount(candidates) < 2) return 0; + if (direct_reads) return candidates; + const near = candidates & (device_nodes orelse candidates); + return if (near != 0) near else candidates; +} + +/// Parses a kernel node list ("0-1", "0,2-3") into a mask; zero on any error. +fn parseNodeList(text: []const u8) u64 { + var mask: u64 = 0; + var ranges = std.mem.tokenizeAny(u8, text, ", \n"); + while (ranges.next()) |range| { + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return 0; + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return 0 + else + first; + if (last < first or ends.next() != null) return 0; + var node = first; + while (node <= last) : (node += 1) { + if (node < 64) mask |= @as(u64, 1) << @intCast(node); + } + } + return mask; +} + +fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 { + return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); +} + +test "automatic NUMA placement keeps the devices' nodes for buffered reads and spreads direct reads over every candidate" { + // One candidate or none: the kernel places. + try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, false)); + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null, false)); + // Devices on one of two nodes: that node; on both: both. + try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, false)); + // Unknown device nodes, or none among the candidates: every candidate. + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, false)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100, false)); + // A cpuset of nodes 0 and 2 of four, devices on node 1: the cpuset's + // nodes, as `pagesNumaMask` narrows the candidates to them first. + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, false)); + + // Direct reads: every candidate, wherever the devices are. + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b10, true)); + try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, true)); + try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, true)); + // Still nothing to choose with one candidate, and still within the + // cpuset. + try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, true)); + try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, true)); +} + +test "parseNodeList accepts kernel node lists" { + try std.testing.expectEqual(@as(u64, 0b11), parseNodeList("0-1\n")); + try std.testing.expectEqual(@as(u64, 0b1101), parseNodeList("0,2-3")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("1-0")); + try std.testing.expectEqual(@as(u64, 0), parseNodeList("x")); + try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0,64")); +} + +test "Workspace capacity counts whole blocks under the mapped ceiling, and carve refuses one more" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const ceiling = Workspace.mapped_bytes_ceiling; + + try std.testing.expectEqual(ceiling / 64, workspace.capacity(64)); + _ = try workspace.allocate(150); + _ = try workspace.allocate(127); + // Whole blocks only: the arenas hold three, their tails none. + try std.testing.expectEqual(3 + (ceiling - 277) / 64, workspace.capacity(64)); + // Capacity is tight, as direct `create` sizes its carve from it: one + // block more crosses the ceiling (reaching it would map 16 GiB). + try std.testing.expectError(error.OutOfMemory, workspace.allocate(ceiling - 277 + 1)); + try std.testing.expectError(error.OutOfMemory, workspace.carve(64, workspace.capacity(64) + 1, 0)); + try std.testing.expectEqual(@as(usize, 277), workspace.mappedBytes()); +} + +test "Workspace carve maps each step's missing blocks as one arena and leaves tails out" { + const allocator = std.testing.allocator; + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + const first = try workspace.allocate(150); + const second = try workspace.allocate(127); + + // Three blocks are mapped: two more make the first five, three more the + // eight. + const blocks = try workspace.carve(64, 5, 8); + try std.testing.expectEqual(@as(usize, 8), blocks.len); + try std.testing.expectEqual(@as(usize, 4), workspace.arenas.items.len); + try std.testing.expectEqual(@as(usize, 2 * 64), workspace.arenas.items[2].bytes.len); + try std.testing.expectEqual(@as(usize, 3 * 64), workspace.arenas.items[3].bytes.len); + for (blocks) |block| try std.testing.expectEqual(@as(usize, 64), block.len); + // Tails stay mapped without contributing a block. + try std.testing.expectEqual(first.ptr, blocks[0].ptr); + try std.testing.expectEqual(first.ptr + 64, blocks[1].ptr); + try std.testing.expectEqual(second.ptr, blocks[2].ptr); +} + +test "Workspace arena ownership cleans up allocation failures" { + const AllocationTest = struct { + fn run(allocator: Allocator) !void { + var workspace: Workspace = .initForTesting(allocator, std.testing.io); + defer workspace.deinit(); + + _ = try workspace.allocate(64); + _ = try workspace.allocate(128); + // The first step is already mapped, as calibration's ring covers + // one device's reserve; the second maps the last block. + const blocks = try workspace.carve(64, 3, 4); + try std.testing.expectEqual(@as(usize, 4), blocks.len); + try std.testing.expectEqual(@as(usize, 256), workspace.mappedBytes()); + try std.testing.expectEqual(@as(usize, 3), workspace.arenas.items.len); + } + }; + try std.testing.checkAllAllocationFailures(std.testing.allocator, AllocationTest.run, .{}); +} diff --git a/zml/io/loader.zig b/zml/io/loader.zig new file mode 100644 index 00000000..7d63c4d8 --- /dev/null +++ b/zml/io/loader.zig @@ -0,0 +1,2888 @@ +//! Checkpoint loading. `Loader.start` takes a whole model: it starts +//! compiling the programs that compute some of its tensors +//! (`Source.bindProgram`) and publishes every tensor the loader fills +//! itself. `wait` then publishes the programs' inputs in groups the devices +//! have room for, runs each group's programs on the caller's task once its +//! inputs landed, and retires everything. Backends only read sources into +//! buffers; lookup, compilation, admission and execution stay in this +//! shared front end. The contract is in the pub doc comments below; +//! docs/learn/loader.md is the user guide. +//! +//! From bindings to specs. `start` checks every tensor and program input +//! once, before anything is published (`prepareSpec`, `checkPlacement`). +//! It lowers each bound source to bytes (`Source.Placed.lower`, the one +//! rule), one part per piece: a part of the destination's logical bytes +//! (the row-major bytes of its shape, before the sharding splits them), a +//! run of `width` bytes repeated over strided dimensions. A whole +//! checkpoint tensor is one part. +//! A strided part of at most `max_part_runs` runs, or of runs of at least +//! `large_run_bytes`, becomes one part per run; a spec with any other +//! strided part (row interleaves, transposes, broadcasts) goes to the +//! backend as strided parts, which the direct backend gathers. The parts go +//! in destination order, and the bytes none covers are zero. Each +//! `LoadSpec` is one destination, which the backends trust as +//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces +//! are disjoint by construction: nothing here checks that (Source's tests +//! do). +//! +//! Order and admission. The tensors the loader fills go out first, as one +//! submission, so every program group is admitted with them counted. A +//! group holds the tensors of consecutive programs in model order, up to +//! `program_group_bytes` of outputs, so the inputs of one layer's experts, +//! adjacent in the checkpoint, are read together. Before publishing a +//! group, `wait` waits for its programs to compile, then retires the oldest +//! pending submissions until the group fits the room every device reports +//! (`admission`): its limit, minus what is in use, minus the bytes the +//! direct backend published but has not allocated yet, minus a 64 MiB +//! reserve. A device's allocator may count freed memory as in use until it +//! is given back (`Platform.releaseFreedDeviceMemory`), so `start` gives +//! back what was freed before the load. A group costs its inputs, its +//! outputs and the largest of its programs' temporaries per device, since +//! they run one after the other. Once no program group is pending, retiring +//! frees nothing more: the room is read again after the retired groups' +//! memory is given back, and the group goes even if it is still short, +//! since the room is an estimate and can read low. A group that does not +//! fit a device fails its allocation or its execution with the device's +//! error, which `wait` returns; a smaller `Options.program_group_bytes` +//! makes smaller groups. Where a device reports no limit (CPU) or the +//! backend does not count its allocations (buffered), nothing is measured +//! and one program group is in flight at a time. +const std = @import("std"); + +const pjrt = @import("pjrt"); +const VFS = @import("vfs"); + +const Buffer = @import("../buffer.zig").Buffer; +const Compiler = @import("../Compiler.zig"); +const Exe = @import("../exe.zig").Exe; +const mem = @import("../mem.zig"); +const Bufferized = mem.Bufferized; +const meta = @import("../meta.zig"); +const ops = @import("../ops.zig"); +const platform_mod = @import("../platform.zig"); +const Memory = TensorStore.Memory; +const Platform = platform_mod.Platform; +const safetensors = @import("../safetensors.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const testing = @import("../testing.zig"); +const backend = @import("backend.zig"); +const Backend = backend.Backend; +const buffered_loader = @import("buffered_loader.zig"); +const LoadPart = backend.LoadPart; +const LoadSpec = backend.LoadSpec; +const DeviceBytes = backend.DeviceBytes; +const direct_loader = @import("direct_loader.zig"); +const load_log = @import("log.zig").load; +const ShardGrid = @import("ShardGrid.zig"); +const TensorStore = @import("TensorStore.zig"); + +const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; + +/// Loads models from their checkpoint sources. A value: `init` returns at +/// once. +/// +/// What it borrows. The platform outlives the loader and every buffer it +/// loads. `io`, and the VFS behind it, outlive the loader: its tasks run on +/// `io`, and the direct backend opens each source file through it and +/// keeps it open until `deinit`, under a copy of the file's path, so +/// nothing of the store is kept past `wait`. `start` borrows the store, the +/// buffers and the progress node until `wait` or `cancel` returns, and +/// reads the model during the call only. The loaded buffers are the +/// caller's and outlive the loader. +/// +/// Start and wait serially on the owning task; source reads and transfers +/// run concurrently inside the backend, compiles on tasks of their own, or +/// inline when `io` cannot run one concurrently. Submissions are retired in +/// publish order. +/// +/// What fails. Every error of `start` and `wait` fails the loader: +/// everything started is canceled (`cancel`), and every later call returns +/// the first error. Each is logged once, at err, where it is detected, but +/// for a plain allocation failure (`OutOfMemory` from an allocator) and +/// `Canceled`, the caller's own `cancel`, which are returned unlogged. A +/// model whose bindings are wrong is `InvalidBinding`, logged with the +/// tensor's field path, the key its source reads (none for a source that +/// reads nothing or a run inside a byte) and the rule it breaks, or for a +/// program whose output differs, the program's name and the shape it +/// computes; buffers shaped unlike the model (`mem.pairBufferized`) too, +/// logged with the field path where they differ. The other errors a caller +/// can act on: a missing checkpoint file is `FileNotFound`; a denied one +/// `AccessDenied` locally and `PermissionDenied` remotely; a file that ends +/// inside a tensor `UnexpectedEndOfFile`, because it changed after its +/// header was read; host memory the backend cannot allocate, pin or fit +/// under its mapped ceiling `OutOfMemory`; device memory that runs out +/// PJRT's `ResourceExhausted`; an io that cannot run the loader's tasks +/// concurrently `ConcurrencyUnavailable`; `cancel` `Canceled`. The rest +/// (PJRT, the compiler, a remote that failed) passes through as its source +/// names it. +pub const Loader = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + /// `init`'s task, which warms the device allocators and creates the + /// backend, until the first `start` awaits it. + setup: ?std.Io.Future(anyerror!Backend), + /// What `setup` created; null before the first `start`, and after a + /// failed set-up. + backend: ?Backend = null, + /// `Options.program_group_bytes`. + program_group_bytes: usize, + /// The programs of the loads started since the last `wait`, each + /// compiled once. + programs: std.ArrayList(*CompiledProgram) = .empty, + /// Program groups started and not yet published, oldest first. + queued: std.Deque(Group) = .empty, + /// Submissions published and not yet retired, oldest first. + pending: std.Deque(PendingSubmission) = .empty, + /// The first error; every later call returns it. + failure: ?anyerror = null, + /// The direct backend when admission can measure the room: it counts + /// its allocations and every device reported an allocator limit at the + /// first `start`. Otherwise one program group is in flight at a time. + measured: ?*direct_loader.Loader = null, + stats: Stats = .{}, + + pub const Options = struct { + /// The checkpoint's path or URI, read by `init` only, which copies a + /// local one the VFS may read direct for the set-up task + /// (`backend.Options.checkpoint`). When `io` is a VFS, the VFS + /// backend serving it sizes the reads (`VFS.readHints`), and the + /// direct backend decides at set-up how such a local one's files + /// read, and places its pinned memory for that + /// (`direct_loader.Loader.create`). Null, or a plain `io`: local + /// sizing, pinned memory near the devices. + checkpoint: ?[]const u8 = null, + /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for + /// the load. Null: 16 for a local checkpoint, 32 for a high-latency + /// one. The direct backend may lower it to fit its pinned memory + /// (logged at debug). + read_parallelism: ?usize = null, + /// The direct backend's DMA block, at most 32 MiB, and a multiple of + /// `VFS.direct_io_alignment` when `io` is a VFS and `direct_io` is + /// not `off`. Null: calibration picks one of + /// `dma_calibration.default_block_sizes`, which suits every measured + /// host. A tuning knob, ignored by the buffered backend; `init` + /// refuses a block the checkpoint's reads cannot use. + dma_block_size: ?usize = null, + /// Direct I/O for local source files, decided once per file per + /// loader (`VFS.DirectIo` for the page-cache trade-off). Nothing + /// changes when `io` is not a VFS. + direct_io: VFS.DirectIo = .auto, + /// Output bytes of the program tensors one group holds at most, one + /// tensor at least. A group's inputs are read together and freed + /// once its programs ran. + program_group_bytes: usize = 8 << 30, + + /// The default read widths by latency class, from the recorded width + /// sweeps. Local sources knee at 12 to 16 on every host measured: the + /// B70 peaks at 12 to 16 and loses 11% at 24, one MI300X prefers 12, + /// the RTX 5090 host 16 over 12, and the GB300 plateau from 12 to 48 + /// has 16 within 5% of its best rung, so 16 is the width with the + /// smallest regret. Remote sources plateau from 24 to 32: hf:// reads + /// 32 and 64 alike and AWS was flat from 24, while pinned memory grows + /// with the width (32 x 32 MiB requests pin 1.5 GiB). + const local_read_parallelism: usize = 16; + const high_latency_read_parallelism: usize = 32; + }; + + /// Concurrent source reads at most (`Options.read_parallelism`). + pub const max_read_parallelism: usize = 128; + + /// Checks `opts` and returns at once. Warming the device allocators and + /// creating the backend (for the direct one: DMA calibration and the + /// pinned pre-growth) run on a task of the loader's own, or here when + /// `io` has no concurrency; the first `start` waits for that task and + /// returns its error. A read width or a block size out of range, a + /// block that is not a multiple of the direct-I/O alignment + /// (`Options.dma_block_size`), a checkpoint whose scheme the VFS does + /// not serve, a block calibration may select whose request (the read + /// chunk rounded up to whole blocks) passes 32 MiB, or a request too + /// short for a widened direct read is `InvalidOptions`, logged. A copy + /// of the path of a local checkpoint the VFS may read direct + /// (`Options.checkpoint`) that cannot be allocated is `OutOfMemory`, + /// unlogged. + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + ) error{ InvalidOptions, OutOfMemory }!Loader { + var backend_opts = try backendOptions(io, opts); + if (backend_opts.checkpoint) |path| backend_opts.checkpoint = try allocator.dupe(u8, path); + const setup_args = .{ allocator, io, platform, backend_opts }; + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .setup = io.concurrent(setUp, setup_args) catch io.async(setUp, setup_args), + .program_group_bytes = opts.program_group_bytes, + }; + } + + /// `init`'s task: the device allocators warmed, then the backend. It + /// owns `opts.checkpoint`, `init`'s copy, and runs to its end even when + /// `deinit` cancels it. + fn setUp(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: backend.Options) anyerror!Backend { + defer if (opts.checkpoint) |path| allocator.free(path); + platform.warmupDeviceAllocators(io) catch |err| { + load_log.failure(err, "warm the device allocators", .{}); + return err; + }; + return Backend.init(allocator, io, platform, opts); + } + + /// Waits for `init`'s task the first time: its backend, then whether + /// admission can measure the devices; or its error, for the caller to + /// fail the loader with. + fn awaitSetup(self: *Loader) !void { + const setup = if (self.setup) |*task| task else return; + const result = setup.await(self.io); + self.setup = null; + self.backend = try result; + self.measured = self.probeMemory(); + } + + /// Gives up on everything started: drops the queued groups, aborts the + /// backend (queued reads are dropped, reads in flight finish), retires + /// the pending submissions without running their programs, and waits + /// for the compiles. Each buffer of the started loads is then empty or + /// holds its shards, whose contents are unspecified (unwritten, errored, + /// or partly written); read none, and free the tree as after a load + /// (`mem.deinitBufferized`). The loader is failed afterwards. For an + /// error between `start` and `wait`; nothing to do when nothing was + /// started. + pub fn cancel(self: *Loader) void { + if (self.loadsStarted()) self.fail(error.Canceled); + } + + /// Cancels `init`'s task when no `start` awaited it, destroying what it + /// built, and destroys the backend: its host memory, its workers and the + /// files it keeps open, closed through `io`. The buffers it loaded stay + /// the caller's. Panics on loads neither waited for nor canceled. + pub fn deinit(self: *Loader) void { + if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); + self.stats.logAdmission(self.measured != null); + self.queued.deinit(self.allocator); + self.pending.deinit(self.allocator); + self.programs.deinit(self.allocator); + if (self.setup) |*setup| { + if (setup.cancel(self.io)) |backend_| backend_.destroy() else |_| {} + } + if (self.backend) |backend_| backend_.destroy(); + self.* = undefined; + } + + /// Whether loads are started and neither waited for nor canceled. + fn loadsStarted(self: *const Loader) bool { + return self.queued.len != 0 or self.pending.len != 0 or self.programs.items.len != 0; + } + + /// The direct backend when every device reports an allocator limit + /// (`measured`). Decided once: the CPU plugin logs an unimplemented + /// warning on every stats call, so admission never asks a device that + /// answered nothing. + fn probeMemory(self: *const Loader) ?*direct_loader.Loader { + const direct = switch (self.backend.?) { + .direct => |direct| direct, + .buffered => return null, + }; + for (self.platform.devices) |device| { + if (device.memoryStats().bytes_limit == null) return null; + } + return direct; + } + + /// Starts loading every tensor of `model` into its buffer in `buffers`, + /// as `store` binds it, and returns; `wait` returns once it is done. + /// Each tensor is placed by its shape's sharding (replicated without a + /// mesh), into the memory its binding names, assembled from the pieces + /// of its source (`Source.Placed.bind`) with every byte no piece covers + /// set to zero. All of these are published here, as one submission. A + /// tensor a program computes (`Source.bindProgram`) starts its program + /// compiling, once for every tensor of an equal program over the same + /// placed shapes; `wait` lands its inputs and runs it. + /// + /// Every tensor and program input is checked once, before anything is + /// published: a tensor `store` did not bind, a source that reads no + /// bytes (an empty checkpoint tensor among them), a source that this + /// loader cannot assemble (a run that starts or ends inside a byte, + /// strided parts into host memory, a slice along an inner axis, a + /// broadcast or a transpose whose source range does not fit the direct + /// backend's smallest read: the checkpoint's read chunk (8 MiB locally), + /// rounded up to whole blocks of the smallest it may pick, 2 MiB or a + /// pinned `dma_block_size`), a sub-byte last dimension that does not + /// pack into whole bytes, a placement its mesh does not divide or on + /// devices the platform lacks, and a host kind the platform cannot take + /// are `error.InvalidBinding`. + /// + /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A + /// tree of another structure (other union arms, optionals or slice + /// lengths: `mem.pairBufferized`) is `InvalidBinding`, logged with the + /// field path, before any buffer is written. `start` empties them + /// again, and each stays empty until its load writes it whole. So + /// after any error each buffer is empty or holds shards for the caller + /// to free: `mem.deinitBufferized` frees the tree either way. `buffers` + /// and `store` are borrowed until `wait` or `cancel` returns; do not end + /// `progress` before then. `start` adds one item to `progress` for each + /// tensor the loader fills, a composite one once, and for each program + /// input; the backend completes them. + /// Several loads may be started before one `wait`, from one store or + /// several. The first `start` waits for `init`'s task, and fails with + /// its error. Each `start` then gives the device memory freed before it + /// back to the devices (`Platform.releaseFreedDeviceMemory`). + pub fn start( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + if (self.failure) |err| return err; + self.startModel(model, buffers, store, progress) catch |err| { + self.fail(err); + return err; + }; + } + + fn startModel( + self: *Loader, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), + store: *const TensorStore, + progress: std.Progress.Node, + ) !void { + try self.awaitSetup(); + // Give back what was freed before this load (compile scratch, the + // set-up's calibration buffers), so that admission's room does not + // count it and the load's peak does not include it. + self.platform.releaseFreedDeviceMemory(); + // The load's time starts once the set-up is done and the freed + // memory given back. + self.stats.begin(self.io); + var prepared = try prepareModel(self.allocator, self.platform, self.backend.?, store, model, buffers); + errdefer prepared.arena.deinit(); + var items = prepared.specs.len; + for (prepared.computed) |item| items += item.compute.inputs.len; + progress.increaseEstimatedTotalItems(items); + // Queued first: their programs compile while the rest is read. + try self.queueGroups(store, prepared.computed, progress); + if (prepared.specs.len == 0) { + prepared.arena.deinit(); + return; + } + try self.submit(prepared.specs, prepared.arena, &.{}, @splat(0), progress); + } + + /// Publishes the queued program groups, oldest first, each once its + /// programs compiled and the devices have room for it (`admit`), and + /// retires every submission in publish order, running a group's + /// programs on this task once its inputs landed. Returns once every + /// started load is in its buffers, with the freed device memory, its + /// programs' inputs and temporaries among it, given back to the devices + /// (`Platform.releaseFreedDeviceMemory`), and logs one info line: the + /// bytes loaded, the time since the first `start` and their rate, then + /// what the programs computed and how long publishing waited for their + /// compiles. A program that fails to compile or run fails it; one that + /// does not return its tensor's shape, placed alike, is + /// `InvalidBinding`; a group that does not fit a device fails it with + /// the device's error. Idempotent: with nothing started it returns at + /// once and logs nothing. + pub fn wait(self: *Loader) !void { + if (self.failure) |err| return err; + const started = self.loadsStarted(); + self.drain() catch |err| { + self.fail(err); + return err; + }; + self.releasePrograms(); + if (started) self.platform.releaseFreedDeviceMemory(); + } + + fn drain(self: *Loader) !void { + var groups: usize = 0; + var tensors: usize = 0; + while (self.queued.popFront()) |group| { + tensors += group.calls.len; + try self.publish(group); + groups += 1; + } + while (self.pending.len != 0) try self.retireOldest(true); + self.stats.end(self.io, tensors, self.programs.items.len, groups); + } + + /// Records `err` unless the loader failed already, and gives up on + /// everything started (`cancel`). The one writer of `failure`. + fn fail(self: *Loader, err: anyerror) void { + if (self.failure == null) self.failure = err; + while (self.queued.popFront()) |group| group.deinit(self.allocator); + // Nothing still pending can succeed. + if (self.pending.len != 0) self.backend.?.abort(); + while (self.pending.len != 0) self.retireOldest(false) catch {}; + self.releasePrograms(); + } + + /// Queues the tensors of `computed`, in model order, in groups of at + /// most `program_group_bytes` of outputs, one tensor at least. + fn queueGroups(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) !void { + var first: usize = 0; + while (first < computed.len) { + var end = first + 1; + var bytes = computed[first].shape.byteSize(); + while (end < computed.len and bytes + computed[end].shape.byteSize() <= self.program_group_bytes) : (end += 1) { + bytes += computed[end].shape.byteSize(); + } + try self.queued.ensureUnusedCapacity(self.allocator, 1); + self.queued.pushBackAssumeCapacity(try self.prepareGroup(store, computed[first..end], progress)); + first = end; + } + } + + /// The group computing `computed`: each tensor's program, the input + /// shells it runs over and the specs that fill them, one per input, + /// checked as `start` checks every spec (`prepareSpec`) before the + /// program's compile starts, which packs their shapes. + fn prepareGroup(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) PrepareError!Group { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + errdefer arena.deinit(); + const calls = try self.allocator.alloc(Call, computed.len); + var prepared: usize = 0; + errdefer { + for (calls[0..prepared]) |*call| call.deinit(self.allocator); + self.allocator.free(calls); + } + var specs: std.ArrayList(LoadSpec) = .empty; + for (computed, calls) |item, *call| { + const inputs = try self.allocator.alloc(Buffer, item.compute.inputs.len); + errdefer self.allocator.free(inputs); + for (item.compute.inputs, inputs, 0..) |source, *input, index| { + input.* = .{ ._platform = self.platform, ._shape = source.shape, ._shards = .empty }; + const target: Target = .{ .path = item.path, .shape = item.shape, .input = .{ .index = index, .shape = source.shape } }; + const spec = try prepareSpec(arena.allocator(), self.platform, self.backend.?, target, source, .default, input); + try specs.append(arena.allocator(), spec); + } + const compiled = try self.compiledProgram(store, item.compute, item.shape); + call.* = .{ .path = try arena.allocator().dupe(u8, item.path), .program = compiled, .inputs = inputs, .output = item.output }; + prepared += 1; + } + return .{ .calls = calls, .specs = specs.items, .arena = arena, .progress = progress }; + } + + /// The compiled program computing a tensor of `output` from `compute`: + /// one started already for an equal program over the same placed + /// shapes, else a compile started now. + fn compiledProgram(self: *Loader, store: *const TensorStore, compute: TensorStore.Binding.Compute, output: Shape) std.mem.Allocator.Error!*CompiledProgram { + for (self.programs.items) |compiled| { + if (compiled.compute.sameExecutable(compiled.output, compute, output)) return compiled; + } + try self.programs.ensureUnusedCapacity(self.allocator, 1); + const compiled = try CompiledProgram.start(self.allocator, self.io, self.platform, store.meshes, compute, output); + self.programs.appendAssumeCapacity(compiled); + return compiled; + } + + fn releasePrograms(self: *Loader) void { + for (self.programs.items) |compiled| compiled.destroy(self.allocator, self.io); + self.programs.clearRetainingCapacity(); + } + + /// Publishes `group` once its programs compiled and the devices have + /// room for it: its calls and its specs' arena move to the pending + /// submission, or are freed when this fails. + fn publish(self: *Loader, group: Group) !void { + errdefer group.deinit(self.allocator); + const started: std.Io.Timestamp = .now(self.io, .awake); + for (group.calls) |call| try call.program.await(self.io, self.platform); + self.stats.compile_wait_ns +|= @intCast(started.untilNow(self.io, .awake).toNanoseconds()); + const execution = try self.admit(group.calls); + try self.submit(group.specs, group.arena, group.calls, execution, group.progress); + } + + /// Retires pending submissions, oldest first, until the group of + /// `calls` fits the measured room or no program group is pending: a + /// bulk load frees nothing. Once nothing can be retired for it, the + /// room is read again after the freed device memory is given back + /// (`Platform.releaseFreedDeviceMemory`), and the group is admitted + /// whatever its size (`Stats.beyond_room`). Without memory measurement + /// one program group is in flight at a time. Returns the group's output + /// and temporary bytes per device, zero while unmeasured. + fn admit(self: *Loader, calls: []const Call) !DeviceBytes { + var inputs: DeviceBytes = @splat(0); + var execution: DeviceBytes = @splat(0); + var room: ?DeviceBytes = null; + if (self.measured) |direct| { + var temporaries: DeviceBytes = @splat(0); + for (calls) |call| { + const exe = &call.program.exe.?; + for (exe.input_shapes) |shape| addPlacementBytes(&inputs, self.platform, shape); + // The output's memory space is the program's to choose; it is + // charged to the devices like the default space. + addPlacementBytes(&execution, self.platform, exe.output_shapes[0]); + // The group's programs run one after the other, so only the + // largest temporaries are live at once. + const temp = self.compiledTempBytes(exe); + const output_mesh = self.platform.placedShape(exe.output_shapes[0])._sharding.mesh.?; + for (output_mesh.devicesInCanonicalOrder()) |device| temporaries[device.id] = @max(temporaries[device.id], temp); + } + for (&execution, temporaries) |*bytes, temp| bytes.* +|= temp; + room = self.readRoom(direct); + } + var retired: usize = 0; + while (room == null or !self.fits(&room.?, &inputs, &execution)) : (retired += 1) { + if (!self.programGroupPending()) { + // Nothing is left to retire, but the retired groups' inputs + // may still count as in use: read the room again once the + // devices have them back. The group then goes, beyond the + // room if the room is still short. + if (room != null) { + self.platform.releaseFreedDeviceMemory(); + room = self.readRoom(self.measured.?); + } + break; + } + try self.retireOldest(true); + if (self.measured) |direct| room = self.readRoom(direct); + } + const beyond_room = room != null and !self.fits(&room.?, &inputs, &execution); + self.stats.admission_retires += retired; + if (beyond_room) self.stats.beyond_room += 1; + if (retired != 0 or beyond_room) { + // Unmeasured, the group's bytes are not computed: only the retires are logged. + if (self.measured != null) { + const n = self.platform.devices.len; + // A device that stopped reporting its limit shows no room. + const rooms: DeviceBytes = room orelse @splat(0); + const pending = self.pendingExecution(); + // The device the fit is decided on: the short one when the + // group goes beyond the room. + const device = admission.tightest(rooms[0..n], pending[0..n], inputs[0..n], execution[0..n]); + load_log.debug("execute admission: retired={d} before submission {d}: measured=true, beyond_room={}, device {d} room={Bi:.2} pending_execution={Bi:.2} inputs={Bi:.2} execution={Bi:.2}", .{ + retired, + self.stats.submissions, + beyond_room, + device, + rooms[device], + pending[device], + inputs[device], + execution[device], + }); + } else { + load_log.debug("execute admission: retired={d} before submission {d}: measured=false", .{ retired, self.stats.submissions }); + } + } + return execution; + } + + /// Whether a group of `inputs` and `execution` fits beside the pending + /// program groups in `room`, on every device. + fn fits(self: *const Loader, room: *const DeviceBytes, inputs: *const DeviceBytes, execution: *const DeviceBytes) bool { + const n = self.platform.devices.len; + const pending = self.pendingExecution(); + return admission.admits(room[0..n], pending[0..n], inputs[0..n], execution[0..n]); + } + + /// Output and temporary bytes of the pending program groups, per + /// device: what retiring them will take before it frees anything. + fn pendingExecution(self: *const Loader) DeviceBytes { + var sum: DeviceBytes = @splat(0); + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + for (&sum, submission.execution) |*bytes, more| bytes.* +|= more; + } + return sum; + } + + /// Whether a program group is pending: only retiring one frees room + /// for the next (a bulk load's buffers stay). + fn programGroupPending(self: *const Loader) bool { + var it = self.pending.iterator(); + while (it.nextPtr()) |submission| { + if (submission.calls.len != 0) return true; + } + return false; + } + + /// The room each device reports now; null when one reports no memory + /// limit. + fn readRoom(self: *Loader, direct: *const direct_loader.Loader) ?DeviceBytes { + const n = self.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + var room: DeviceBytes = undefined; + for (room[0..n], self.platform.devices, unallocated[0..n]) |*out, device, device_unallocated| { + const reported = device.memoryStats(); + out.* = admission.room(reported.bytes_limit, reported.bytes_in_use, device_unallocated, admission.reserve_bytes) orelse return null; + self.stats.min_room_seen = @min(self.stats.min_room_seen orelse out.*, out.*); + } + return room; + } + + /// Temporaries the executable takes on each of its devices, from its + /// compiled memory stats (per partition, i.e. per device). Zero when + /// the plugin cannot answer. + fn compiledTempBytes(self: *Loader, exe: *const Exe) u64 { + const api = self.platform.pjrt_api; + const pjrt_executable = exe.exe.executable(api) catch |err| return self.tempUnavailable(err); + defer pjrt_executable.deinit(api); + const stats = pjrt_executable.getCompiledMemoryStats(api) catch |err| return self.tempUnavailable(err); + return stats.temp_size_in_bytes; + } + + fn tempUnavailable(self: *Loader, err: pjrt.ApiError) u64 { + if (!self.stats.temp_unavailable_logged) { + self.stats.temp_unavailable_logged = true; + load_log.debug("executable memory stats unavailable ({s}): temporaries counted as zero", .{@errorName(err)}); + } + return 0; + } + + /// One submission over `specs`, which live in `arena`. Takes `arena` and + /// `calls` once the submission is published; on failure the caller still + /// owns them. `execution` is a program group's output and temporary + /// bytes per device (`admit`), zero for a bulk load. + fn submit( + self: *Loader, + specs: []const LoadSpec, + arena: std.heap.ArenaAllocator, + calls: []Call, + execution: DeviceBytes, + progress: std.Progress.Node, + ) !void { + // One spec per destination, however many parts fill it: its bytes + // count once. + var logical_bytes: usize = 0; + for (specs) |spec| logical_bytes += spec.shape.byteSize(); + try self.pending.ensureUnusedCapacity(self.allocator, 1); + const submission = try self.backend.?.submit(specs, progress); + self.pending.pushBackAssumeCapacity(.{ + .submission = submission, + .arena = arena, + .calls = calls, + .execution = execution, + .logical_bytes = logical_bytes, + }); + self.stats.submissions += 1; + } + + /// Retires the oldest pending submission: waits for its reads and + /// transfers, runs its programs when `execute`, frees its inputs either + /// way and counts its bytes once they ran. + fn retireOldest(self: *Loader, execute: bool) !void { + var oldest = self.pending.popFront().?; + defer self.release(&oldest); + try self.backend.?.await(oldest.submission); + if (!execute) return; + for (oldest.calls) |*call| { + call.execute(self.allocator, self.io) catch |err| { + const target: Target = .{ .path = call.path, .shape = call.program.output }; + load_log.failure(err, "run program {s} computing {f}: inputs={d}", .{ call.program.name(), target, call.inputs.len }); + return err; + }; + } + self.stats.bytes_loaded += oldest.logical_bytes; + } + + fn release(self: *Loader, oldest: *PendingSubmission) void { + oldest.arena.deinit(); + for (oldest.calls) |*call| call.deinit(self.allocator); + self.allocator.free(oldest.calls); + } +}; + +/// The loader's counters: read only by logs and tests; nothing here +/// decides. +const Stats = struct { + /// When the first `start` since the last `wait` ran: `wait` logs the + /// load's time from there. + started_at: ?std.Io.Timestamp = null, + /// Logical bytes of the submissions retired with execution since that + /// `start`: each destination's shape once, so a composite tensor counts + /// its padding and not its sources one by one, and a program group + /// counts its inputs. + bytes_loaded: usize = 0, + /// How long publishing waited for compiles since the last `wait`. + compile_wait_ns: u64 = 0, + /// Submissions published since `init`. + submissions: usize = 0, + /// Submissions retired to admit a program group, bulk loads included. + admission_retires: usize = 0, + /// The least room a device reported since `init`; null while nothing + /// was measured. + min_room_seen: ?u64 = null, + /// Program groups admitted beyond the measured room since `init`: + /// nothing was left to retire for them. + beyond_room: usize = 0, + /// Whether `tempUnavailable` logged that the plugin gives no compiled + /// memory stats: once per loader. + temp_unavailable_logged: bool = false, + + /// Times and counts a load from the first `start` since the last + /// `wait`; later starts add to it. + fn begin(self: *Stats, io: std.Io) void { + if (self.started_at != null) return; + self.started_at = .now(io, .awake); + self.bytes_loaded = 0; + } + + /// Logs `wait`'s info line, unless nothing was started, and ends the + /// load's timing. `programs` programs computed `tensors` tensors in + /// `groups` groups. + fn end(self: *Stats, io: std.Io, tensors: usize, programs: usize, groups: usize) void { + const started_at = self.started_at orelse return; + const took = started_at.untilNow(io, .awake); + const seconds = @as(f64, @floatFromInt(@max(took.nanoseconds, 1))) / std.time.ns_per_s; + const rate: u64 = @intFromFloat(@as(f64, @floatFromInt(self.bytes_loaded)) / seconds); + if (groups == 0) { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); + } else { + load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s], computed {d} tensors with {d} programs in {d} groups, {d} ms waiting for compiles", .{ + self.bytes_loaded, + took, + rate, + tensors, + programs, + groups, + self.compile_wait_ns / std.time.ns_per_ms, + }); + } + self.started_at = null; + self.compile_wait_ns = 0; + } + + /// `deinit`'s debug line. `measured`: whether admission measured the + /// room (`Loader.measured`), printed as `memory_supported`. + fn logAdmission(self: *const Stats, measured: bool) void { + if (self.submissions == 0) return; + load_log.debug("loader admission: submissions={d}, execute_admission_retires={d}, beyond_room={d}, memory_supported={}, min_room_seen={Bi:.2}, reserve={Bi:.2}", .{ + self.submissions, + self.admission_retires, + self.beyond_room, + measured, + self.min_room_seen orelse 0, + admission.reserve_bytes, + }); + } +}; + +/// One published submission: what to wait for, what to run afterwards and +/// what its accounting owes. +const PendingSubmission = struct { + submission: backend.Submission, + /// The specs, which the backend reads until the submission is awaited + /// (`backend.LoadSpec`). + arena: std.heap.ArenaAllocator, + /// Run in order once the reads are done; freed with their inputs when + /// the submission is retired. Empty for a bulk load. + calls: []Call, + /// Output and temporary bytes per device of a program group, while + /// admission measures (`admit`); zero otherwise and for a bulk load. + execution: DeviceBytes, + logical_bytes: usize, +}; + +/// Program tensors whose inputs `wait` publishes as one submission. +const Group = struct { + calls: []Call, + /// One per program input, filling the calls' input shells; they and + /// their parts live in `arena`. + specs: []LoadSpec, + arena: std.heap.ArenaAllocator, + progress: std.Progress.Node, + + fn deinit(self: Group, allocator: std.mem.Allocator) void { + for (self.calls) |*call| call.deinit(allocator); + allocator.free(self.calls); + self.arena.deinit(); + } +}; + +/// A program compiling, then compiled. +const CompiledProgram = struct { + compute: TensorStore.Binding.Compute, + output: Shape, + future: std.Io.Future(Compiler.Error!Exe), + /// Set by the first `await`. + exe: ?Exe = null, + + fn start( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + meshes: []const *const Sharding.Mesh, + compute: TensorStore.Binding.Compute, + output: Shape, + ) std.mem.Allocator.Error!*CompiledProgram { + const self = try allocator.create(CompiledProgram); + // Concurrent, as `io.async` may run the compile right here on a busy + // threaded io, ahead of the reads `start` publishes after it. + const args = .{ compute, allocator, io, platform, meshes, output }; + const compile = TensorStore.Binding.Compute.compile; + self.* = .{ + .compute = compute, + .output = output, + .future = io.concurrent(compile, args) catch io.async(compile, args), + }; + return self; + } + + /// Waits for the compile the first time, then checks that the + /// executable computes one tensor of `output`, placed alike + /// (`checkOutput`). + fn await(self: *CompiledProgram, io: std.Io, platform: *const Platform) !void { + if (self.exe != null) return; + self.exe = self.future.await(io) catch |err| { + load_log.failure(err, "compile program {s} computing {f}", .{ self.name(), self.output }); + return err; + }; + try checkOutput(platform, self.name(), self.output, &self.exe.?); + } + + fn name(self: *const CompiledProgram) []const u8 { + return self.compute.program.name(); + } + + fn destroy(self: *CompiledProgram, allocator: std.mem.Allocator, io: std.Io) void { + if (self.exe) |*exe| { + exe.deinit(); + } else if (self.future.cancel(io)) |exe| { + exe.deinit(); + } else |_| {} + allocator.destroy(self); + } +}; + +/// One tensor of a program group, a pending `Exe.call`: its program, the +/// input shells the group's submission fills, and the model's buffer it +/// computes. +const Call = struct { + /// The tensor's field path, in the group's arena: `prepareModel`'s + /// arena may be freed before the call runs. + path: []const u8, + program: *CompiledProgram, + inputs: []Buffer, + output: *Buffer, + + /// Runs the program's executable, awaited when the group was + /// published, over the loaded inputs on the calling task. + /// `Exe.call` leaves every input handle to the caller, donated ones + /// included (their contents are gone), so `deinit` frees them all, + /// exactly once; a runner would destroy the donated handles itself. + /// A failed execution, an input whose transfer failed included, is + /// returned and leaves `output` unwritten. + fn execute(self: *const Call, allocator: std.mem.Allocator, io: std.Io) !void { + const exe = &self.program.exe.?; + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{self.inputs}); + try exe.call(io, args, &results, .{ .wait = true }); + self.output.* = results.get(Buffer); + } + + /// Frees the inputs, donated or not; a shell the loader never wrote owns + /// no shards. + fn deinit(self: *Call, allocator: std.mem.Allocator) void { + for (self.inputs) |*input| input.deinit(); + allocator.free(self.inputs); + } +}; + +/// The arithmetic behind program admission: what the devices have left, +/// and whether the next group fits beside the ones still pending. +/// Everything is per device (index = `platform.devices` index) in bytes; +/// nothing here touches a device, so the rules are tested without one. +const admission = struct { + /// Allocator slack per device. BFC rounds every allocation up to 256 bytes; + /// a 64-pack, 16-source, 8-device load is about 1e5 shards, under 32 MiB of + /// rounding, and nothing else runs on the devices during a load. + const reserve_bytes: u64 = 64 << 20; + + /// Bytes the loader may still take on one device: the limit (null when the + /// plugin reports none, as CPU does) minus what is in use, minus its own + /// bytes published but not allocated yet (`unallocated`), minus the + /// reserve. + fn room(limit: ?u64, in_use: u64, unallocated: u64, reserve: u64) ?u64 { + return (limit orelse return null) -| in_use -| unallocated -| reserve; + } + + /// Whether one submission's `inputs` (placements, counted here until they + /// are published) and `execution` (output placement plus the executable's + /// temporaries) fit beside the pending executions on every device. + fn admits(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) bool { + for (rooms, pending, inputs, execution) |r, p, i, e| { + if (p +| i +| e > r) return false; + } + return true; + } + + /// The device with the least room left once one submission's `inputs` + /// and `execution` sit beside the pending executions: one that `admits` + /// fails on when the submission does not fit. `rooms` is not empty. + fn tightest(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) usize { + var device: usize = 0; + var least: i128 = std.math.maxInt(i128); + for (rooms, pending, inputs, execution, 0..) |r, p, i, e, index| { + const left = @as(i128, r) - (p +| i +| e); + if (left < least) { + least = left; + device = index; + } + } + return device; + } +}; + +test "room subtracts use, unallocated bytes and the reserve, saturating" { + try std.testing.expectEqual(null, admission.room(null, 0, 0, 0)); + try std.testing.expectEqual(55, admission.room(100, 30, 10, 5).?); + // More in use than the limit admits: no room, no underflow. + try std.testing.expectEqual(0, admission.room(100, 130, 0, 0).?); + try std.testing.expectEqual(0, admission.room(100, 90, 0, 20).?); +} + +test "admits needs room on every device" { + try std.testing.expect(admission.admits(&.{ 40, 40 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // The second device is one byte short. + try std.testing.expect(!admission.admits(&.{ 40, 34 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + try std.testing.expect(admission.admits(&.{}, &.{}, &.{}, &.{})); +} + +test "tightest names the device short of room, not the first" { + // Device 0 has less room but holds the group; device 1 is one byte short. + try std.testing.expectEqual(1, admission.tightest(&.{ 30, 34 }, &.{ 0, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); + // Both are short: the one short by more, though it has more room. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 12 }, &.{ 0, 0 }, &.{ 10, 20 }, &.{ 5, 10 })); + // A need that saturates at maxInt(u64) is short by the most. + try std.testing.expectEqual(1, admission.tightest(&.{ 10, 20 }, &.{ 0, std.math.maxInt(u64) }, &.{ 20, 1 }, &.{ 0, 0 })); + // Equally short: the first. + try std.testing.expectEqual(0, admission.tightest(&.{ 10, 10 }, &.{ 0, 0 }, &.{ 12, 12 }, &.{ 0, 0 })); +} + +/// Adds the bytes `shape` occupies per device: its box on each device it +/// is placed on, as the direct backend's `countSealed` counts them; an +/// empty shape takes none. `shape` places on the platform's devices +/// (`checkPlacement`). +fn addPlacementBytes(out: *DeviceBytes, platform: *const Platform, shape: Shape) void { + if (shape.byteSize() == 0) return; + const grid: ShardGrid = .init(platform, shape); + for (grid.devices) |device| out[device.id] += grid.box_bytes; +} + +/// A tensor a program computes, at `path` in the model, the buffer it +/// computes into, and its program and inputs. +const Computed = struct { + path: []const u8, + shape: Shape, + output: *Buffer, + compute: TensorStore.Binding.Compute, +}; + +/// A model as `start` takes it: one spec per tensor the loader fills, and +/// the tensors programs compute, all in `arena` with the specs' parts and +/// the tensors' field paths. +const PreparedModel = struct { + arena: std.heap.ArenaAllocator, + specs: []const LoadSpec, + computed: []const Computed, +}; + +/// Pairs each tensor of `model` with its buffer and binding: a tensor the +/// loader fills takes one spec (`prepareSpec`), a tensor a program computes +/// is placed like any other (`checkPlacement`) and computed. +fn prepareModel( + allocator: std.mem.Allocator, + platform: *const Platform, + backend_: Backend, + store: *const TensorStore, + model: anytype, + buffers: *Bufferized(@TypeOf(model.*)), +) PrepareError!PreparedModel { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + // Tensors and buffers pair by structure, before any buffer is written: a + // buffer tree shaped otherwise is refused whole. + var report_buffer: [256]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = mem.pairBufferized(arena.allocator(), model, buffers, &report) catch |err| switch (err) { + error.Mismatch => { + load_log.err("buffers shaped unlike the model at model{s}; bufferize them from this model", .{report.buffered()}); + return error.InvalidBinding; + }, + error.OutOfMemory => return error.OutOfMemory, + }; + // Empty until a backend or a program writes it whole, so the caller can + // free the tree whatever fails. + for (pairs) |pair| pair.buffer.* = .{ ._platform = platform, ._shape = pair.tensor.shape(), ._shards = .empty }; + + var specs: std.ArrayList(LoadSpec) = try .initCapacity(arena.allocator(), pairs.len); + var computed: std.ArrayList(Computed) = try .initCapacity(arena.allocator(), pairs.len); + for (pairs) |pair| { + const tensor = pair.tensor; + const target: Target = .{ .path = pair.path, .shape = tensor.shape() }; + const binding = store.bindingOf(tensor.id) orelse { + load_log.err("{f} has no checkpoint binding", .{target}); + return error.InvalidBinding; + }; + switch (binding) { + .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, backend_, target, load.source, load.memory, pair.buffer)), + .compute => |compute| { + try checkPlacement(platform, target, compute.program.name()); + computed.appendAssumeCapacity(.{ .path = pair.path, .shape = tensor.shape(), .output = pair.buffer, .compute = compute }); + }, + } + } + return .{ .arena = arena, .specs = specs.items, .computed = computed.items }; +} + +/// A strided part with few runs, or with large ones, expands into one part +/// per run, each read and transferred on its own: at most `max_part_runs` +/// runs, or runs of at least `large_run_bytes`. A spec with any other +/// strided part goes to the backend whole, as strided parts +/// (`backend.Lattice`), which the direct backend reads by destination +/// ranges. +const max_part_runs = 64; +const large_run_bytes = 1 << 20; + +fn expandable(lattice: backend.Lattice) bool { + return lattice.dims.len == 0 or lattice.runs() <= max_part_runs or lattice.width >= large_run_bytes; +} + +/// What a spec fills, as the logs name it: the model's tensor at `path` +/// (`mem.Pair.path`) of `shape`, or `input` of the program computing it. +const Target = struct { + path: []const u8, + shape: Shape, + input: ?Input = null, + + const Input = struct { index: usize, shape: Shape }; + + /// The shape the spec fills: the input's, else the tensor's. + fn destination(self: Target) Shape { + return if (self.input) |input| input.shape else self.shape; + } + + pub fn format(self: Target, writer: *std.Io.Writer) !void { + if (self.input) |input| try writer.print("program input {d} {f} of ", .{ input.index, input.shape }); + try writer.print("model{s} {f}", .{ self.path, self.shape }); + } +}; + +/// The spec filling `output`, the buffer of `target`, from `source` into +/// `memory`, its parts in `arena`: the source lowered to bytes +/// (`Source.Placed.lower`), expanded into its runs unless a part does not +/// expand, in destination order. The one check of what the backends trust +/// (`backend.LoadSpec`, `Backend.stridedRead`); a rule it breaks is +/// `InvalidBinding`, logged with the target, and with the key its source +/// reads once the source is lowered. +fn prepareSpec( + arena: std.mem.Allocator, + platform: *const Platform, + backend_: Backend, + target: Target, + source: TensorStore.Source.Placed, + memory: Memory, + output: *Buffer, +) PrepareError!LoadSpec { + const shape = target.destination(); + const lowered = source.lower(arena) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.SubBytePosition => { + load_log.err("{f}: its source {f} reads a run that starts or ends inside a byte, which the loader cannot place", .{ target, source.shape }); + return error.InvalidBinding; + }, + }; + if (lowered.len == 0) { + load_log.err("{f} reads no checkpoint bytes", .{target}); + return error.InvalidBinding; + } + const key = lowered[0].source.name; + var expands = true; + var runs: u64 = 0; + for (lowered) |part| { + expands = expands and expandable(part.lattice); + runs += part.lattice.runs(); + } + try checkPlacement(platform, target, key); + if (!backend.supportsMemory(platform, memory)) { + load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); + return error.InvalidBinding; + } + if (!expands and memory.isHost()) { + load_log.err("{f} from {s}: strided parts into {t} memory, which this loader does not assemble", .{ target, key, memory }); + return error.InvalidBinding; + } + const parts = if (!expands) lowered else expanded: { + if (runs == lowered.len) break :expanded lowered; + const expanded = try arena.alloc(LoadPart, @intCast(runs)); + var next: usize = 0; + for (lowered) |part| { + var cursor: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); + while (cursor.next()) |run| : (next += 1) { + expanded[next] = .{ .source = part.source, .lattice = .{ .src_offset = run.src, .dst_offset = run.dst, .width = part.lattice.width } }; + } + } + break :expanded expanded; + }; + std.mem.sort(LoadPart, parts, {}, struct { + fn lessThan(_: void, lhs: LoadPart, rhs: LoadPart) bool { + return lhs.lattice.dst_offset < rhs.lattice.dst_offset; + } + }.lessThan); + if (!backend.checkParts(parts, shape.byteSize())) { + load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); + return error.InvalidBinding; + } + // After `checkParts`, which bounds the offsets `stridedRead` adds. + for (parts) |part| { + const strided = backend_.stridedRead(part); + if (strided.read > strided.limit) { + load_log.err("{f} from {s}: a strided part spans {d} source bytes, more than one read of {d}", .{ target, key, strided.read, strided.limit }); + return error.InvalidBinding; + } + } + return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; +} + +/// Checks that `target`, from `from` (its source's key or its program), +/// places on `platform`'s devices, in any memory space: a sub-byte last +/// dimension packs into whole bytes (`Shape.packedShape`), its mesh +/// partitions divide its dimensions, and every device of its mesh is one of +/// the platform's. +fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) error{InvalidBinding}!void { + const shape = target.destination(); + const bits = shape.dtype().bitSizeOf(); + if (bits < 8 and (shape.rank() == 0 or @rem(shape.dim(-1), 8 / bits) != 0)) { + load_log.err("{f} from {s}: its last dimension does not pack into whole bytes", .{ target, from }); + return error.InvalidBinding; + } + const placed = platform.placedShape(shape).packedShape(); + const placement = Sharding.Placement.init(&placed) catch { + load_log.err("{f} from {s}: its dimensions are not divisible by its mesh partitions", .{ target, from }); + return error.InvalidBinding; + }; + for (placement.mesh().?.devicesInCanonicalOrder()) |device| { + if (device.id >= platform.devices.len) { + load_log.err("{f} from {s} is placed on device {d}, but {t} has {d}", .{ target, from, device.id, platform.target, platform.devices.len }); + return error.InvalidBinding; + } + } +} + +/// `opts` as the backends take them, the checkpoint resolved to the read +/// hints of the VFS backend serving it and the read width to its default, +/// and borrowed when it is local and may read direct +/// (`backend.Options.checkpoint`). Every option is checked here, once. +fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backend.Options { + const hints: VFS.ReadHints = hints: { + const path = opts.checkpoint orelse break :hints .local; + const vfs = VFS.fromIo(io) orelse break :hints .local; + break :hints vfs.readHints(path) catch |err| { + load_log.err("invalid loader options: checkpoint={s}: {s}", .{ path, @errorName(err) }); + return error.InvalidOptions; + }; + }; + if (opts.read_parallelism) |width| { + if (width == 0 or width > Loader.max_read_parallelism) { + load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ width, Loader.max_read_parallelism }); + return error.InvalidOptions; + } + } + const max_request = backend.max_read_request_size; + if (hints.read_chunk_size == 0 or hints.read_chunk_size > max_request) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, expected 1..{d}: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, max_request }); + return error.InvalidOptions; + } + var options: backend.Options = .{ + .read_parallelism = opts.read_parallelism orelse if (hints.high_latency) + Loader.Options.high_latency_read_parallelism + else + Loader.Options.local_read_parallelism, + .read_hints = hints, + .dma_block_size = opts.dma_block_size, + .direct_io = opts.direct_io, + }; + // Checked for every block calibration may select. + const alignment = backend.readAlignment(io, opts.direct_io); + for (options.dmaBlockSizes()) |block_size| { + if (backend.requestSize(hints.read_chunk_size, block_size, alignment) == null) { + load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, dma_block_size={d}, alignment={d}, max_request={d}: no request size fits: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, block_size, alignment, max_request }); + return error.InvalidOptions; + } + } + // A nonzero alignment means a VFS that may read a local file direct. + // `isLocal` opens nothing. + const path = opts.checkpoint orelse return options; + if (alignment != 0 and VFS.fromIo(io).?.isLocal(path)) options.checkpoint = path; + return options; +} + +/// A program must compute one tensor of `expected`, placed on the same +/// devices, each holding the same slice. The comparison is per device, not +/// by mesh pointer and partition, so `.open` and `.replicated`, or a null +/// mesh and the replicated mesh, agree when they place alike. `expected` +/// places on the platform's devices (`checkPlacement`). +fn checkOutput(platform: *const Platform, name: []const u8, expected: Shape, exe: *const Exe) error{InvalidBinding}!void { + if (exe.output_shapes.len != 1 or !expected.eql(exe.output_shapes[0])) { + load_log.err("program {s} computing {f} returns {d} outputs, the first {f}", .{ name, expected, exe.output_shapes.len, if (exe.output_shapes.len != 0) exe.output_shapes[0] else expected }); + return error.InvalidBinding; + } + const actual = exe.output_shapes[0]; + const want_shape = platform.placedShape(expected).packedShape(); + const want = Sharding.Placement.init(&want_shape) catch |err| + std.debug.panic("loader: {f} does not place, though checked: {s}", .{ expected, @errorName(err) }); + const got_shape = platform.placedShape(actual).packedShape(); + const got = Sharding.Placement.init(&got_shape) catch |err| { + load_log.err("program {s}: placement of its output {f}: {s}", .{ name, actual, @errorName(err) }); + return error.InvalidBinding; + }; + const want_devices = want.mesh().?.devicesInCanonicalOrder(); + const got_devices = got.mesh().?.devicesInCanonicalOrder(); + if (want_devices.len != got_devices.len) { + load_log.err("program {s} places its output {f} on {d} devices, the tensor {f} on {d}", .{ name, actual, got_devices.len, expected, want_devices.len }); + return error.InvalidBinding; + } + for (want_devices, got_devices, 0..) |w, g, device_index| { + const want_slices = want.slices(device_index, want_shape); + const got_slices = got.slices(device_index, got_shape); + const same_slices = want_slices.len == got_slices.len and for (want_slices.constSlice(), got_slices.constSlice()) |a, b| { + if (a.start != b.start or a.size != b.size) break false; + } else true; + if (w.id != g.id or !same_slices) { + load_log.err("program {s} places its output {f} unlike the tensor {f} on device {d}", .{ name, actual, expected, w.id }); + return error.InvalidBinding; + } + } +} + +/// A four-byte tensor `value`, a four-byte `second`, a `missing` entry in a +/// file that does not exist, an `empty` tensor and a CPU platform. Pinned +/// after `init`. +const LoaderTestFixture = struct { + const contents = [_]u8{ 1, 2, 3, 4 }; + const second_contents = [_]u8{ 5, 6, 7, 8 }; + + tmp: std.testing.TmpDir, + store: TensorStore, + platform: *const Platform, + value: Tensor, + second: Tensor, + missing: Tensor, + empty: Tensor, + + fn init(self: *LoaderTestFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + const file = try self.tmp.dir.createFile(io, "weights.bin", .{ .read = true }); + try file.writePositionalAll(io, &(contents ++ second_contents), 0); + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + const path = path_buffer[0..path_len]; + var missing_buffer: [1024]u8 = undefined; + const missing_path = try std.fmt.bufPrint(&missing_buffer, "{s}.missing", .{path}); + + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ + .file_uri = path, + .name = "value", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "second", + .shape = .init(.{second_contents.len}, .u8), + .offset = contents.len, + }); + try registry.registerTensor(.{ + .file_uri = missing_path, + .name = "missing", + .shape = .init(.{contents.len}, .u8), + .offset = 0, + }); + try registry.registerTensor(.{ + .file_uri = path, + .name = "empty", + .shape = .init(.{0}, .u8), + .offset = 0, + }); + self.platform = testing.env(); + self.store = .init(allocator, ®istry, .{ .meshes = &.{self.platform.replicated_mesh} }); + self.value = self.store.view().createReplicatedTensor("value", null); + self.second = self.store.view().createReplicatedTensor("second", null); + self.missing = self.store.view().createReplicatedTensor("missing", null); + self.empty = self.store.view().createReplicatedTensor("empty", null); + } + + fn deinit(self: *LoaderTestFixture) void { + self.store.deinit(); + self.tmp.cleanup(); + } + + /// A tensor of `like`'s shape that `program` computes from the + /// checkpoint tensor `name`. + fn computed(self: *LoaderTestFixture, name: []const u8, like: Tensor, program: anytype) Tensor { + const input = self.store.view().source(name, null).withPartitioning(.replicated, .{}); + return TensorStore.Source.bindProgram(&.{input}, like.shape(), program); + } + + /// The program returning its one input. + const Identity = struct { + pub fn forward(_: Identity, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0]; + } + }; + + /// The program adding `value` to every byte of its one input. + const Add = struct { + value: u8, + + pub fn forward(self: Add, _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].add(Tensor.scalar(self.value, .u8)); + } + }; + + /// Whether `loader` failed with `expected` and holds nothing started. + fn expectFailed(loader: *const Loader, expected: anyerror) !void { + try std.testing.expectEqual(expected, loader.failure.?); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); + } +}; + +test "start publishes the bulk at once and wait runs program groups one at a time without memory measurement" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // One program tensor per group. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); + defer loader.deinit(); + // Exercise serial admission regardless of the platform's memory stats: + // decided once the set-up is done. + try loader.awaitSetup(); + loader.measured = null; + + const Model = struct { value: Tensor, first: Tensor, second: Tensor }; + const model: Model = .{ + .value = fixture.value, + .first = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + .second = fixture.computed("second", fixture.second, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // The bulk is published; the groups wait for `wait`, their one program + // (the same function and shapes) compiling meanwhile. + try std.testing.expectEqual(1, loader.pending.len); + try std.testing.expectEqual(2, loader.queued.len); + try std.testing.expectEqual(1, loader.programs.items.len); + try loader.wait(); + try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); + // Serial admission retired the bulk and the first group before it + // published the second. + try std.testing.expectEqual(2, loader.stats.admission_retires); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.first, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.second, &LoaderTestFixture.second_contents, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); + + // Idempotent: a second wait neither reruns nor recounts. + try loader.wait(); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); + + // The next load counts and times from its own first start. + try std.testing.expectEqual(null, loader.stats.started_at); + var again: Buffer = undefined; + try loader.start(&fixture.second, &again, &fixture.store, .none); + try loader.wait(); + defer again.deinit(); + try std.testing.expectEqual(LoaderTestFixture.second_contents.len, loader.stats.bytes_loaded); + try std.testing.expectEqual(null, loader.stats.started_at); +} + +test "one loader takes loads from different stores before one wait" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var other_store = try testing.storeOf(allocator, &fixture.store.registry, .{ .meshes = &.{fixture.platform.replicated_mesh} }); + defer other_store.deinit(); + const other = other_store.view().createReplicatedTensor("second", null); + + var first: Buffer = undefined; + var second: Buffer = undefined; + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&fixture.value, &first, &fixture.store, .none); + try loader.start(&other, &second, &other_store, .none); + try loader.wait(); + defer first.deinit(); + defer second.deinit(); + try testing.expectShards(allocator, io, &first, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &second, &LoaderTestFixture.second_contents, .default); +} + +test "tensors of one program and equal args share one compile, and one group runs them all" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Add = LoaderTestFixture.Add; + const Model = struct { a: Tensor, b: Tensor, c: Tensor }; + const model: Model = .{ + .a = fixture.computed("value", fixture.value, Add{ .value = 1 }), + .b = fixture.computed("second", fixture.second, Add{ .value = 1 }), + .c = fixture.computed("value", fixture.value, Add{ .value = 2 }), + }; + try std.testing.expect(fixture.store.bindingOf(model.a.id).? == .compute); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Nothing for the loader to fill itself, and one group. + try std.testing.expectEqual(0, loader.pending.len); + try std.testing.expectEqual(1, loader.queued.len); + try std.testing.expectEqual(2, loader.programs.items.len); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.a, &.{ 2, 3, 4, 5 }, .default); + try testing.expectShards(allocator, io, &buffers.b, &.{ 6, 7, 8, 9 }, .default); + try testing.expectShards(allocator, io, &buffers.c, &.{ 3, 4, 5, 6 }, .default); + try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); +} + +test "loader delivers host bindings into pinned host memory without charging the devices" { + for ([_]Memory{ .host_pinned, .host_mapped }) |kind| try expectHostDelivery(kind); +} + +fn expectHostDelivery(kind: Memory) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, kind); + try std.testing.expectEqual(kind, fixture.store.bindingOf(pinned.id).?.load.memory); + + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { device: Tensor, host: Tensor }; + const model: Model = .{ .device = fixture.value, .host = pinned }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + if (!backend.supportsMemory(platform, kind)) { + // Refused before anything is submitted. + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + return; + } + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.device, &LoaderTestFixture.contents, .default); + // One shard per device, each in that device's pinned host memory space + // or, `host_mapped`, a view of pinned memory in its default space. + try testing.expectShards(allocator, io, &buffers.host, &LoaderTestFixture.second_contents, kind); + + // Only the tensor in device memory counts against the devices, on both + // sides of the admission ledger. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + for (direct.sealed_bytes[0..platform.devices.len], direct.allocated_bytes[0..platform.devices.len]) |sealed, *allocated| { + try std.testing.expectEqual(LoaderTestFixture.contents.len, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + } + } +} + +test "a program frees its donated and non-donated inputs once" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Sum = struct { + donate: bool, + + pub fn forward(self: @This(), _: Shape, inputs: []const Tensor) Tensor { + const sum = inputs[0].add(inputs[1]); + return if (self.donate) sum.reuseBuffer(inputs[0]) else sum; + } + }; + for ([_]bool{ false, true }) |donate| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const tensor = TensorStore.Source.bindProgram(&.{ + store.view().source("value", null).withPartitioning(.replicated, .{}), + store.view().source("weight", null).withPartitioning(.replicated, .{}), + }, shape, Sum{ .donate = donate }); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &store, .none); + errdefer loader.cancel(); + try loader.programs.items[0].await(io, platform); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), loader.programs.items[0].exe.?.donated_input_indices.len); + // Retiring runs the program, then frees every input handle once, the + // donated one included. + try loader.wait(); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + } +} + +test "cancel gives up on everything started without running its programs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expectEqual(@as(usize, 0), buffers.computed._shards.len); + // The loader is failed: what it waits for and what it is given. + try std.testing.expectError(error.Canceled, loader.wait()); + try std.testing.expectError(error.Canceled, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "cancel frees a published group's inputs without running its program" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + const Model = struct { computed: Tensor }; + const model: Model = .{ .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + // Publish the group as `wait` does, then give up before it retires. + try loader.publish(loader.queued.popFront().?); + try std.testing.expect(loader.programGroupPending()); + loader.cancel(); + try LoaderTestFixture.expectFailed(&loader, error.Canceled); + try std.testing.expect(!loader.programGroupPending()); + try std.testing.expectEqual(0, buffers.computed._shards.len); +} + +test "a read failure fails the loader for every later call" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // The broken group first, retired when the next one is admitted. + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); + defer loader.deinit(); + try loader.awaitSetup(); + loader.measured = null; + + const Model = struct { broken: Tensor, value: Tensor }; + const model: Model = .{ + .broken = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + .value = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(@as(usize, 0), buffers.value._shards.len); + try std.testing.expectError(error.FileNotFound, loader.start(&model, &buffers, &fixture.store, .none)); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); +} + +test "after a failed load every buffer is empty or whole, and deinitBufferized frees the tree" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tree with a slice and optionals, some of it in the file that loads, + // a tensor in the file that does not exist, and a program over it. + const Layer = struct { weight: Tensor, bias: ?Tensor }; + const Model = struct { layers: []Layer, broken: Tensor, computed: Tensor }; + var layers = [_]Layer{ + .{ .weight = fixture.value, .bias = fixture.second }, + .{ .weight = fixture.second, .bias = null }, + }; + const model: Model = .{ + .layers = &layers, + .broken = fixture.missing, + .computed = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), + }; + var buffers = try mem.bufferize(allocator, &model); + // Frees whatever the failed load wrote, under the testing allocator. + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + // Each buffer holds no shard or one per device. + const devices = fixture.platform.placedShape(fixture.value.shape())._sharding.mesh.?.devicesInCanonicalOrder().len; + var torn: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shards: usize, count: *usize) void { + if (buffer._shards.len != 0 and buffer._shards.len != shards) count.* += 1; + } + }.call, .{ devices, &torn }); + try std.testing.expectEqual(0, torn); + try std.testing.expectEqual(0, buffers.computed._shards.len); +} + +test "the direct backend allocates what it was submitted once everything landed" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var placed: DeviceBytes = @splat(0); + // An empty tensor takes no bytes. + addPlacementBytes(&placed, fixture.platform, .init(.{ 0, 4 }, .f32)); + addPlacementBytes(&placed, fixture.platform, fixture.value.shape()); + for (placed[0..fixture.platform.devices.len]) |bytes| try std.testing.expectEqual(LoaderTestFixture.contents.len, bytes); + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + const Model = struct { value: Tensor, computed: Tensor }; + const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + try loader.wait(); + if (loader.backend.? != .direct) return; + const direct = loader.backend.?.direct; + const n = fixture.platform.devices.len; + const unallocated = direct.unallocatedBytes(); + // Replicated program inputs and bulk outputs occupy every device. + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n], unallocated[0..n]) |sealed, *allocated, bytes| { + try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, sealed); + try std.testing.expectEqual(sealed, allocated.load(.acquire)); + try std.testing.expectEqual(0, bytes); + } +} + +test "init checks every option and returns InvalidOptions" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + const Case = struct { io: std.Io, opts: Loader.Options }; + const cases = [_]Case{ + .{ .io = io, .opts = .{ .read_parallelism = 0 } }, + .{ .io = io, .opts = .{ .read_parallelism = Loader.max_read_parallelism + 1 } }, + .{ .io = io, .opts = .{ .dma_block_size = 0 } }, + .{ .io = io, .opts = .{ .dma_block_size = backend.max_read_request_size + 1 } }, + // No backend of the VFS serves the scheme. + .{ .io = vfs.io(), .opts = .{ .checkpoint = "missing://bucket/model" } }, + }; + for (cases) |case| { + if (Loader.init(allocator, case.io, platform, case.opts)) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } + // Under a VFS, a local file may be read directly: a block is a whole + // number of alignment units, reads landing in blocks. + if (VFS.direct_io_alignment != 0) { + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "/model.safetensors", .dma_block_size = (8 << 20) + VFS.direct_io_alignment / 2 })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + // Whatever the checkpoint: a request of one alignment unit cannot + // hold a direct read widened at each end. + try vfs.registerBackend("tiny", .{ .io = io, .read_hints = .{ .read_chunk_size = VFS.direct_io_alignment } }); + if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "tiny://model", .dma_block_size = VFS.direct_io_alignment })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidOptions, err); + } + // A plain io has no VFS to ask: local sizing. Nothing started, so + // `deinit` cancels the set-up or destroys what it built. + var loader = try Loader.init(allocator, io, platform, .{ .checkpoint = "missing://bucket/model" }); + loader.deinit(); +} + +test "init copies a local checkpoint's path for its set-up" { + if (VFS.direct_io_alignment == 0) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const unit: [VFS.direct_io_alignment]u8 = @splat(7); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = &unit }); + var root_buffer: [std.fs.max_path_bytes]u8 = undefined; + const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; + var path_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors", .{root}); + + // The copy is the only allocation of `init`: nothing is left to free + // when it fails. + var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = 0 }); + if (Loader.init(failing.allocator(), vfs.io(), platform, .{ .checkpoint = path })) |value| { + var loader = value; + loader.deinit(); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.OutOfMemory, err); + + // The set-up frees `init`'s copy once, whether it ran to its end or + // `deinit` canceled it before any `start`. + var loader = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path }); + defer loader.deinit(); + try loader.awaitSetup(); + for ([_]VFS.DirectIo{ .on, .auto }) |policy| { + var unstarted = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path, .direct_io = policy }); + unstarted.deinit(); + } +} + +test "a failed set-up fails the first start, and deinit frees the rest" { + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, io); + defer fixture.deinit(); + // `init` allocates nothing; the set-up's third allocation, inside the + // backend, fails, so the backend unwinds what it built before it. + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 2 }); + var loader = try Loader.init(failing.allocator(), io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try std.testing.expectError(error.OutOfMemory, loader.start(&fixture.value, &output, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.OutOfMemory); + try std.testing.expect(loader.backend == null); + try std.testing.expectError(error.OutOfMemory, loader.wait()); +} + +test "without concurrency init sets up in place" { + const allocator = std.testing.allocator; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, std.testing.io); + defer fixture.deinit(); + // The testing io, but `io.concurrent` refuses and `io.async` runs the + // function in place. + var vtable = std.testing.io.vtable.*; + vtable.concurrent = struct { + fn concurrent(_: ?*anyopaque, _: usize, _: std.mem.Alignment, _: []const u8, _: std.mem.Alignment, _: *const fn (*const anyopaque, *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + return error.ConcurrencyUnavailable; + } + }.concurrent; + vtable.async = struct { + fn async(_: ?*anyopaque, result: []u8, _: std.mem.Alignment, context: []const u8, _: std.mem.Alignment, start: *const fn (*const anyopaque, *anyopaque) void) ?*std.Io.AnyFuture { + start(context.ptr, result.ptr); + return null; + } + }.async; + const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; + + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try std.testing.expect(loader.setup.?.any_future == null); + // Nothing started: neither has anything to do. + loader.cancel(); + try loader.wait(); + try std.testing.expectEqual(null, loader.failure); + + var output: Buffer = undefined; + try loader.start(&fixture.value, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); +} + +test "a rate-limited HTTP source loads through the VFS hold" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const tensor_bytes = 1024; + const tensors = 4; + + // Four tensor ranges with a gap between them, so the planner keeps four + // jobs and the load makes four concurrent GETs. + var object: [(2 * tensors - 1) * tensor_bytes]u8 = undefined; + for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 31 +% 7); + + var server = try VFS.MockServer.init(io, &object, .{ + // Two GETs per 100 ms: the four readers are throttled repeatedly. + .throttle = .{ .window = .{ .gets = 2, .per_ms = 100 } }, + }); + defer server.deinit(io); + try server.start(io); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http, .retry = .{ + .max_retries = 5, + .initial_delay = .fromMilliseconds(20), + .max_delay = .fromMilliseconds(20), + .max_hold = .fromSeconds(5), + .throttle_budget = .fromSeconds(30), + } }); + defer http.deinit(); + var vfs: VFS = try .init(allocator, io); + defer vfs.deinit(); + try vfs.registerBackend("http", http.backend()); + const stats = &http.governor.stats; + + var url_buffer: [128]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); + + const platform = testing.env(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + const names = [tensors][]const u8{ "a", "b", "c", "d" }; + for (names, 0..) |name, index| { + try registry.registerTensor(.{ + .file_uri = url, + .name = name, + .shape = .init(.{tensor_bytes}, .u8), + .offset = 2 * index * tensor_bytes, + }); + } + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + + const Model = struct { a: Tensor, b: Tensor, c: Tensor, d: Tensor }; + const model: Model = .{ + .a = store.view().createReplicatedTensor("a", null), + .b = store.view().createReplicatedTensor("b", null), + .c = store.view().createReplicatedTensor("c", null), + .d = store.view().createReplicatedTensor("d", null), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + + var loader = try Loader.init(allocator, vfs.io(), platform, .{ + .checkpoint = url, + .read_parallelism = 4, + }); + // Holding costs time, not credits: the direct backend's `destroy` + // asserts that every lifecycle credit came back with its pinned blocks. + // It checks once its workers stopped, since until then one may hold a + // credit for a claim that finds nothing. + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + inline for (comptime std.meta.fieldNames(Model), 0..) |field_name, index| { + try testing.expectShards(allocator, io, &@field(buffers, field_name), object[2 * index * tensor_bytes ..][0..tensor_bytes], .default); + } + + try server.stop(io); + + // The server rate limited the load, the backend held every reader, and + // no reader burned its retry budget for it. + try std.testing.expect(stats.throttles.load(.acquire) > 0); + try std.testing.expect(stats.holds.load(.acquire) > 0); + try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); + try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); + try std.testing.expectEqual(@as(u64, tensors * tensor_bytes), stats.physical_bytes.load(.acquire)); + + // The checkpoint's VFS backend, not the local hints (8 MiB), sized the + // reads: 16 MiB requests over the CPU's 2 MiB blocks. Its file was read + // exact: a remote file is never direct. + const direct = loader.backend.?.direct; + try std.testing.expectEqual(@as(usize, 16 << 20), direct.plan_config.request_size); + try std.testing.expectEqual(@as(usize, 0), direct.source_slots.get(url).?.alignment orelse 0); +} + +test "a rejected binding or a program of the wrong output fails the loader" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = fixture.platform; + // Two rows of two, not the four bytes the tensor holds. + const Rows = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].reshape(.{ 2, 2 }); + } + }; + // A mesh of devices the platform does not have: the same tensors placed + // on it, in device and host memory, and a program's output. + var foreign_physical = platform.replicated_mesh.physical.*; + const foreign_devices = try allocator.dupe(Sharding.Device, foreign_physical.devices_in_canonical_order); + defer allocator.free(foreign_devices); + for (foreign_devices) |*device| device.id += 100; + foreign_physical.devices_in_canonical_order = foreign_devices; + var foreign_mesh = platform.replicated_mesh.*; + foreign_mesh.physical = &foreign_physical; + const pinned = fixture.store.view().createReplicatedTensor("second", null); + fixture.store.view().setMemory(pinned, .host_pinned); + var foreign = [_]Tensor{ fixture.value, pinned, fixture.value }; + for (&foreign) |*tensor| tensor._shape._sharding.mesh = &foreign_mesh; + // The program places its output replicated, the tensor expects its + // bytes split over the devices: same dims, another placement. It needs + // devices that split four bytes. + var misplaced = fixture.value; + misplaced._shape = Shape.init(.{ .x = 4 }, .u8).withPartitioning(platform.meshes.get("model").?, .{ .x = .model }); + const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; + + // No binding, an empty source or program input, a mesh of other + // devices, and a program of the wrong shape or placement. + const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; + const cases = [_]Case{ + .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, + .{ .tensor = fixture.empty }, + .{ .tensor = fixture.computed("empty", fixture.empty, LoaderTestFixture.Identity{}) }, + .{ .tensor = foreign[0] }, + .{ .tensor = foreign[1] }, + .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, + .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, + .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, + }; + for (cases) |case| { + if (case.skip) continue; + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + if (case.at_wait) { + try loader.start(&case.tensor, &output, &fixture.store, .none); + try std.testing.expectError(error.InvalidBinding, loader.wait()); + } else { + try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); + } + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + // `start` emptied the buffer before it failed. + try std.testing.expectEqual(0, output._shards.len); + output.deinit(); + try std.testing.expectError(error.InvalidBinding, loader.start(&fixture.value, &output, &fixture.store, .none)); + } +} + +test "buffers of another structure fail start before any buffer is written" { + var fixture: LoaderTestFixture = undefined; + try fixture.init(std.testing.allocator, std.testing.io); + defer fixture.deinit(); + const value = fixture.value; + const second = fixture.second; + + // Each pair holds as many tensors on both sides, in another structure, + // and a pair precedes each mismatch. + const Optionals = struct { head: Tensor, a: ?Tensor, b: ?Tensor }; + try expectStructureMismatch(&fixture, Optionals, .{ .head = value, .a = value, .b = null }, .{ .head = value, .a = null, .b = value }); + const Arms = struct { head: Tensor, arm: union(enum) { a: Tensor, b: Tensor }, last: Tensor }; + try expectStructureMismatch(&fixture, Arms, .{ .head = value, .arm = .{ .a = value }, .last = second }, .{ .head = value, .arm = .{ .b = value }, .last = second }); + const Slices = struct { head: Tensor, first: []Tensor, second: []Tensor }; + var two = [_]Tensor{ value, second }; + var one = [_]Tensor{value}; + var other_one = [_]Tensor{second}; + var other_two = [_]Tensor{ second, value }; + try expectStructureMismatch(&fixture, Slices, .{ .head = value, .first = &two, .second = &one }, .{ .head = value, .first = &other_one, .second = &other_two }); +} + +/// Starts `model` into the buffers bufferized from `built`, which has as +/// many tensors in another structure: `InvalidBinding`, and every buffer as +/// it was. +fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, built: Model, model: Model) !void { + const allocator = std.testing.allocator; + var loader = try Loader.init(allocator, std.testing.io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var buffers = try mem.bufferize(allocator, &built); + defer mem.deinitBufferized(allocator, &buffers); + // No tensor has this shape, which `start` would replace with its + // tensor's when it empties a buffer. + const untouched: Shape = .init(.{ 1, 2, 3 }, .u8); + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape) void { + buffer._shape = shape; + } + }.call, .{untouched}); + try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + var written: usize = 0; + meta.forEachVisit(&buffers, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, shape: Shape, count: *usize) void { + if (!buffer._shape.eql(shape) or buffer._shards.len != 0) count.* += 1; + } + }.call, .{ untouched, &written }); + try std.testing.expectEqual(0, written); +} + +/// Two safetensors files for composite loads, every row 16 bytes (8 u16): +/// `first` holds `a` [3, 8] then `b` [2, 8], adjacent, then `spacer`, then +/// `q` [8, 8], `k` [4, 8] and `v` [4, 8]; `second` holds `plain` [4, 8], +/// then `c` [4, 8], the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8] and +/// `wide` [64, 8]. `empty` [0, 8] is registered without bytes. Pinned after +/// `init`. Each store takes a copy of `registry` (`testing.storeOf`). +const CompositeFixture = struct { + tmp: std.testing.TmpDir, + registry: safetensors.TensorRegistry, + a: [48]u8, + b: [32]u8, + c: [64]u8, + plain: [64]u8, + q: [128]u8, + k: [64]u8, + v: [64]u8, + w1: [4][32]u8, + w3: [4][32]u8, + wide: [1024]u8, + + const expert_names = [2][4][]const u8{ + .{ "e0.w1", "e1.w1", "e2.w1", "e3.w1" }, + .{ "e0.w3", "e1.w3", "e2.w3", "e3.w3" }, + }; + + fn init(self: *CompositeFixture, allocator: std.mem.Allocator, io: std.Io) !void { + self.tmp = std.testing.tmpDir(.{}); + errdefer self.tmp.cleanup(); + self.registry = .init(allocator); + errdefer self.registry.deinit(); + testing.pattern(&self.a, 1); + testing.pattern(&self.b, 2); + testing.pattern(&self.c, 3); + testing.pattern(&self.plain, 4); + testing.pattern(&self.q, 9); + testing.pattern(&self.k, 10); + testing.pattern(&self.v, 11); + for (&self.w1, &self.w3, 0..) |*w1, *w3, e| { + testing.pattern(w1, @intCast(20 + e)); + testing.pattern(w3, @intCast(30 + e)); + } + var spacer: [16]u8 = undefined; + testing.pattern(&spacer, 5); + testing.pattern(&self.wide, 6); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "first.safetensors", &self.registry, &.{ + .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &self.a }, + .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.b }, + .{ .name = "spacer", .dtype = "U16", .dims = &.{ 1, 8 }, .bytes = &spacer }, + .{ .name = "q", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &self.q }, + .{ .name = "k", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.k }, + .{ .name = "v", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.v }, + }); + try testing.writeSafetensors(allocator, io, self.tmp.dir, "second.safetensors", &self.registry, &.{ + .{ .name = "plain", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.plain }, + .{ .name = "c", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.c }, + .{ .name = expert_names[0][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[0] }, + .{ .name = expert_names[1][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[0] }, + .{ .name = expert_names[0][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[1] }, + .{ .name = expert_names[1][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[1] }, + .{ .name = expert_names[0][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[2] }, + .{ .name = expert_names[1][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[2] }, + .{ .name = expert_names[0][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[3] }, + .{ .name = expert_names[1][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[3] }, + .{ .name = "wide", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &self.wide }, + }); + try self.registry.registerTensor(.{ + .file_uri = self.registry.tensors.get("a").?.file_uri, + .name = "empty", + .shape = .init(.{ 0, 8 }, .u16), + .offset = 0, + }); + } + + fn deinit(self: *CompositeFixture) void { + self.registry.deinit(); + self.tmp.cleanup(); + } + + fn rows(view: TensorStore.View, name: []const u8) TensorStore.Source { + return view.source(name, .{ .row, .col }); + } +}; + +test "loader assembles concatenated and padded sources on every placement and memory kind" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // a, b and c end to end over 16 rows; c alone over 8 rows; b then a + // over 6 rows. The rest is zero. + var concatenated: [256]u8 = @splat(0); + @memcpy(concatenated[0..48], &fixture.a); + @memcpy(concatenated[48..80], &fixture.b); + @memcpy(concatenated[80..144], &fixture.c); + var padded: [128]u8 = @splat(0); + @memcpy(padded[0..64], &fixture.c); + var reversed: [96]u8 = @splat(0); + @memcpy(reversed[0..32], &fixture.b); + @memcpy(reversed[32..80], &fixture.a); + + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const Model = struct { concatenated: Tensor, padded: Tensor, reversed: Tensor, plain: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + // Rows over the devices (b and c straddle shards on four), every + // row split over them, and every byte on each of them. + inline for (.{ .{ .row = .model }, .{ .col = .model }, .{} }) |partitioning| { + const shards: u64 = if (@typeInfo(@TypeOf(partitioning)).@"struct".field_names.len == 0) 1 else device_count; + for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { + if (!backend.supportsMemory(platform, memory)) continue; + const view = store.view(); + const model: Model = .{ + .concatenated = Source.concatenate(&.{ rows(view, "a"), rows(view, "b"), rows(view, "c") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 7 } }) + .withPartitioning(.tp, partitioning).bind(), + // The padded engram-style table: one source. + .padded = rows(view, "c").pad(0, .{ .row = Source.Pad{ .high = 4 } }).withPartitioning(.tp, partitioning).bind(), + // Out of file order, on the platform's replicated mesh. + .reversed = Source.concatenate(&.{ rows(view, "b"), rows(view, "a") }, .row) + .pad(0, .{ .row = Source.Pad{ .high = 1 } }) + .withPartitioning(.replicated, .{}).bind(), + .plain = rows(view, "plain").withPartitioning(.tp, partitioning).bind(), + }; + inline for (comptime std.meta.fieldNames(Model)) |name| view.setMemory(@field(model, name), memory); + try std.testing.expect(store.bindingOf(model.plain.id).?.load.source.whole() != null); + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated, memory); + try testing.expectShards(allocator, io, &buffers.padded, &padded, memory); + try testing.expectShards(allocator, io, &buffers.reversed, &reversed, memory); + try testing.expectShards(allocator, io, &buffers.plain, &fixture.plain, memory); + // Each destination once, padding included. + try std.testing.expectEqual(@as(usize, 256 + 128 + 96 + 64), loader.stats.bytes_loaded); + // A destination is allocated once per device, whatever its + // parts; host memory counts on neither side. + if (loader.backend.? == .direct) { + const direct = loader.backend.?.direct; + const n = platform.devices.len; + for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n]) |sealed, *allocated| try std.testing.expectEqual(sealed, allocated.load(.acquire)); + for (devices.physical.devices_in_canonical_order) |device| { + const expected: u64 = if (memory.isHost()) 0 else (256 + 128 + 64) / shards + 96; + try std.testing.expectEqual(expected, direct.sealed_bytes[device.id]); + } + } + } + } + } +} + +test "loader assembles interleaved, stacked and split sources" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const rows = CompositeFixture.rows; + const row = 16; + + // Experts with their two projections concatenated, then row by row. + var concatenated_experts: [256]u8 = undefined; + var interleaved_experts: [256]u8 = undefined; + for (0..4) |e| { + @memcpy(concatenated_experts[e * 64 ..][0..32], &fixture.w1[e]); + @memcpy(concatenated_experts[e * 64 + 32 ..][0..32], &fixture.w3[e]); + for (0..2) |r| { + @memcpy(interleaved_experts[e * 64 + r * 2 * row ..][0..row], fixture.w1[e][r * row ..][0..row]); + @memcpy(interleaved_experts[e * 64 + (r * 2 + 1) * row ..][0..row], fixture.w3[e][r * row ..][0..row]); + } + } + + const Model = struct { qkv: Tensor, concatenated: Tensor, interleaved: Tensor, head: Tensor, tail: Tensor }; + for ([_]usize{ 2, 4 }) |device_count| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const platform = devices.platform; + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const groups: i64 = @intCast(device_count); + + // q, k and v cut in one group per device: each device holds its rows + // of every member, contiguous. Every part has `groups` runs. + var members: [3]Source = undefined; + for (&members, [_][]const u8{ "q", "k", "v" }) |*member, name| { + member.* = rows(view, name).splitAxis(.row, .{ .group = groups, .r = -1 }); + } + var experts: [2]Source = undefined; + for (&experts, CompositeFixture.expert_names) |*stacked, names| { + var per_expert: [4]Source = undefined; + for (&per_expert, names) |*expert, name| expert.* = rows(view, name); + stacked.* = Source.stack(&per_expert, 0, .expert); + } + const fused = Source.concatenate(&.{ rows(view, "q"), rows(view, "k") }, .row); + // A part of q, and the rest of q with all of k. + const head, const tail = fused.split(.row, &.{ 4, 8 })[0..2].*; + const model: Model = .{ + .qkv = Source.concatenate(&members, .r).merge(.{ .row = .{ .group, .r } }).withPartitioning(.tp, .{ .row = .model }).bind(), + // Experts are whole rows of the merged axis, one or two per device. + .concatenated = Source.concatenate(&experts, .row) + .merge(.{ .rows = .{ .expert, .row } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .interleaved = Source.stack(&experts, 2, .proj) + .merge(.{ .rows = .{ .expert, .row, .proj } }) + .withPartitioning(.tp, .{ .rows = .model }).bind(), + .head = head.withPartitioning(.replicated, .{}).bind(), + .tail = tail.withPartitioning(.tp, .{ .row = .model }).bind(), + }; + var lowering: std.heap.ArenaAllocator = .init(allocator); + defer lowering.deinit(); + const qkv_parts = try store.bindingOf(model.qkv.id).?.load.source.lower(lowering.allocator()); + try std.testing.expectEqual(@as(usize, 3), qkv_parts.len); + for (qkv_parts) |part| try std.testing.expectEqual(@as(u64, device_count), part.lattice.runs()); + + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var qkv: [256]u8 = undefined; + var at: usize = 0; + for (0..device_count) |g| { + for ([_][]const u8{ &fixture.q, &fixture.k, &fixture.v }) |member| { + const per_group = member.len / device_count; + @memcpy(qkv[at..][0..per_group], member[g * per_group ..][0..per_group]); + at += per_group; + } + } + try testing.expectShards(allocator, io, &buffers.qkv, &qkv, .default); + try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated_experts, .default); + try testing.expectShards(allocator, io, &buffers.interleaved, &interleaved_experts, .default); + try testing.expectShards(allocator, io, &buffers.head, fixture.q[0..64], .default); + var tail_bytes: [128]u8 = undefined; + @memcpy(tail_bytes[0..64], fixture.q[64..128]); + @memcpy(tail_bytes[64..], &fixture.k); + try testing.expectShards(allocator, io, &buffers.tail, &tail_bytes, .default); + } +} + +test "loader gathers strided sources: a row interleave across files, broadcasts and a transpose" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const Source = TensorStore.Source; + // Rows of 16 bytes: g and u interleave row by row, 96 runs each; s0 and + // s1 repeat 100 and 60 times; z [24, 4] is transposed, 96 runs of two + // bytes, then padded with two zero rows. Every part has more runs than + // expand, so the direct backend gathers them, its jobs reading g's rows + // from one file and u's from the other. + var g: [96 * 16]u8 = undefined; + var u: [96 * 16]u8 = undefined; + var z: [24 * 8]u8 = undefined; + var s0: [4]u8 = undefined; + var s1: [4]u8 = undefined; + testing.pattern(&g, 1); + testing.pattern(&u, 2); + testing.pattern(&z, 3); + testing.pattern(&s0, 4); + testing.pattern(&s1, 5); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ + .{ .name = "g", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &g }, + .{ .name = "s0", .dtype = "F32", .dims = &.{}, .bytes = &s0 }, + .{ .name = "z", .dtype = "U16", .dims = &.{ 24, 4 }, .bytes = &z }, + }); + try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ + .{ .name = "s1", .dtype = "F32", .dims = &.{}, .bytes = &s1 }, + .{ .name = "u", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &u }, + }); + + var interleaved: [192 * 16]u8 = undefined; + for (0..96) |r| { + @memcpy(interleaved[2 * r * 16 ..][0..16], g[r * 16 ..][0..16]); + @memcpy(interleaved[(2 * r + 1) * 16 ..][0..16], u[r * 16 ..][0..16]); + } + var scales: [160 * 4]u8 = undefined; + for (0..160) |i| @memcpy(scales[i * 4 ..][0..4], if (i < 100) &s0 else &s1); + var transposed: [6 * 48]u8 = @splat(0); + for (0..4) |column| { + for (0..24) |row| @memcpy(transposed[column * 48 + row * 2 ..][0..2], z[row * 8 + column * 2 ..][0..2]); + } + + const Model = struct { by_rows: Tensor, by_columns: Tensor, scales_by_rows: Tensor, scales_by_columns: Tensor, transposed: Tensor, transposed_by_columns: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const view = store.view(); + const rows = Source.stack(&.{ view.source("g", .{ .row, .col }), view.source("u", .{ .row, .col }) }, 1, .proj) + .merge(.{ .row = .{ .row, .proj } }); + const repeated = Source.concatenate(&.{ + view.source("s0", .{}).broad(Shape.init(.{ .c = 100 }, .f32)), + view.source("s1", .{}).broad(Shape.init(.{ .c = 60 }, .f32)), + }, .c).splitAxis(.c, .{ .row = 40, .col = 4 }); + const flipped = view.source("z", .{ .row, .col }).transpose(.{ .col, .row }).withTags(.{ .row, .col }) + .pad(0, .{ .row = Tensor.Pad{ .high = 2 } }); + const model: Model = .{ + .by_rows = rows.withPartitioning(.tp, .{ .row = .model }).bind(), + .by_columns = rows.withPartitioning(.tp, .{ .col = .model }).bind(), + .scales_by_rows = repeated.withPartitioning(.tp, .{ .row = .model }).bind(), + .scales_by_columns = repeated.withPartitioning(.tp, .{ .col = .model }).bind(), + .transposed = flipped.withPartitioning(.replicated, .{}).bind(), + .transposed_by_columns = flipped.withPartitioning(.tp, .{ .col = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of 128 bytes: every destination takes several. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 256; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.by_rows, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.by_columns, &interleaved, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_rows, &scales, .default); + try testing.expectShards(allocator, io, &buffers.scales_by_columns, &scales, .default); + try testing.expectShards(allocator, io, &buffers.transposed, &transposed, .default); + try testing.expectShards(allocator, io, &buffers.transposed_by_columns, &transposed, .default); + } + } +} + +test "loader gathers a block-scale swizzle, its padding zero" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Scales [e=4, m=31, k=6] in the 128x4 layout at 8x4: m padded to 32 + // and tiled by 2x4, k padded to 8. Its parts repeat runs of four bytes + // or two, more than expand, so the direct backend gathers them. + var scales: [4 * 31 * 6]u8 = undefined; + testing.pattern(&scales, 7); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "scales.safetensors", ®istry, &.{ + .{ .name = "scales", .dtype = "U8", .dims = &.{ 4, 31, 6 }, .bytes = &scales }, + }); + var swizzled: [4 * 32 * 8]u8 = @splat(0); + for (0..4) |e| for (0..4) |mo| for (0..2) |ko| for (0..4) |mi| for (0..2) |m4| for (0..4) |ki| { + const m = mo * 8 + m4 * 4 + mi; + const k = ko * 4 + ki; + const flat = (((mo * 2 + ko) * 4 + mi) * 2 + m4) * 4 + ki; + if (m < 31 and k < 6) swizzled[e * 256 + flat] = scales[(e * 31 + m) * 6 + k]; + }; + + const Model = struct { replicated: Tensor, by_rows: Tensor }; + for ([_]usize{ 1, 2, 4 }) |device_count| { + for ([_]usize{ 0, 64 }) |alignment| { + var devices: testing.Devices = undefined; + try devices.init(allocator, io, device_count); + defer devices.deinit(); + const tp = try devices.mesh("tp"); + var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); + defer store.deinit(); + const swizzle = store.view().source("scales", .{ .e, .m, .k }) + .pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) + .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) + .splitAxis(.k, .{ .ko = -1, .ki = 4 }) + .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) + .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); + const model: Model = .{ + .replicated = swizzle.withPartitioning(.replicated, .{}).bind(), + .by_rows = swizzle.withPartitioning(.tp, .{ .e = .model }).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // Jobs of at most 1024 bytes, each reading the 744 source + // bytes whole: the swizzle reorders its runs. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| { + direct.plan_config.request_size = 1024; + direct.reads.alignment = alignment; + }, + .buffered => {}, + } + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.replicated, &swizzled, .default); + try testing.expectShards(allocator, io, &buffers.by_rows, &swizzled, .default); + } + } +} + +test "bindings the loader cannot fill fail it before submission" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + // Three nibbles: whole bytes hold no element boundary past the first two. + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibbles", + .shape = .init(.{3}, .u4), + .offset = 0, + }); + try fixture.registry.registerTensor(.{ + .file_uri = fixture.registry.tensors.get("a").?.file_uri, + .name = "nibble_rows", + .shape = .init(.{ 2, 3 }, .f4e2m1), + .offset = 0, + }); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const view = store.view(); + const rows = CompositeFixture.rows; + + const rejected = [_]Tensor{ + // Nothing to read: an empty tensor padded. + rows(view, "empty").pad(0, .{ .row = Source.Pad{ .high = 8 } }).withPartitioning(.replicated, .{}).bind(), + // Twelve bits: its one run ends inside a byte (no known checkpoint + // has one). + view.createReplicatedTensor("nibbles", null), + // Strided parts into host memory. + host: { + const tensor = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(); + view.setMemory(tensor, .host_pinned); + break :host tensor; + }, + // Three whole bytes, in rows of three nibbles that do not pack, as a + // tensor and as a program input. + view.createReplicatedTensor("nibble_rows", null), + Source.bindProgram(&.{view.source("nibble_rows", null).withPartitioning(.replicated, .{})}, .init(.{3}, .u8), LoaderTestFixture.Identity{}), + // A program input whose transpose reads its 1024 source bytes in + // every job, more than one read. + Source.bindProgram(&.{rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{})}, .init(.{ 8, 64 }, .u16), LoaderTestFixture.Identity{}), + }; + var output: Buffer = undefined; + for (&rejected) |*tensor| { + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + // One read holds half of `wide`; the buffered backend reads any part. + try loader.awaitSetup(); + switch (loader.backend.?) { + .direct => |direct| direct.plan_config.request_size = 512, + .buffered => if (tensor == &rejected[rejected.len - 1]) continue, + } + try std.testing.expectError(error.InvalidBinding, loader.start(tensor, &output, &store, .none)); + try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); + } +} + +test "loader fills gaps, reordered and strided rows, and a transpose" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: CompositeFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const platform = testing.env(); + var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); + defer store.deinit(); + const Source = TensorStore.Source; + const Pad = Source.Pad; + const view = store.view(); + const rows = CompositeFixture.rows; + const a = rows(view, "a"); + + // b's second row past a gap, the rest zero; b then a, each padded; a's + // last row before its first two, one source feeding two parts; a's + // rows every other row, a strided part of few runs, expanded; and a + // transpose, 512 runs of two bytes, gathered. + const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor, transposed: Tensor }; + const model: Model = .{ + .sparse = rows(view, "b").slice(.row, .{ .start = 1 }).pad(0, .{ .row = Pad{ .low = 4, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .padded = Source.concatenate(&.{ + rows(view, "b").pad(0, .{ .row = Pad{ .high = 2 } }), + a.pad(0, .{ .row = Pad{ .high = 1 } }), + }, .row).withPartitioning(.replicated, .{}).bind(), + .swapped = Source.concatenate(&.{ a.slice(.row, .{ .start = 2 }), a.slice(.row, .{ .end = 2 }) }, .row) + .pad(0, .{ .row = Pad{ .high = 5 } }).withPartitioning(.replicated, .{}).bind(), + .strided = a.pad(0, .{ .row = Pad{ .interior = 1, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), + .transposed = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + try loader.start(&model, &buffers, &store, .none); + try loader.wait(); + + var expected: [128]u8 = @splat(0); + @memcpy(expected[64..80], fixture.b[16..32]); + try testing.expectShards(allocator, io, &buffers.sparse, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..32], &fixture.b); + @memcpy(expected[64..112], &fixture.a); + try testing.expectShards(allocator, io, &buffers.padded, &expected, .default); + expected = @splat(0); + @memcpy(expected[0..16], fixture.a[32..48]); + @memcpy(expected[16..48], fixture.a[0..32]); + try testing.expectShards(allocator, io, &buffers.swapped, &expected, .default); + expected = @splat(0); + for (0..3) |i| @memcpy(expected[i * 32 ..][0..16], fixture.a[i * 16 ..][0..16]); + try testing.expectShards(allocator, io, &buffers.strided, &expected, .default); + var transposed_bytes: [1024]u8 = undefined; + for (0..8) |column| { + for (0..64) |row| @memcpy(transposed_bytes[column * 128 + row * 2 ..][0..2], fixture.wide[row * 16 + column * 2 ..][0..2]); + } + try testing.expectShards(allocator, io, &buffers.transposed, &transposed_bytes, .default); +} + +test "a program takes its inputs from sources, in order" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + const Source = TensorStore.Source; + const view = fixture.store.view(); + + // value's first half then second's last half, assembled by the loader, + // and second as it is: the program subtracts the second input from the + // first, which tells them apart. + const Sub = struct { + pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { + return inputs[0].sub(inputs[1]); + } + }; + const assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x); + const second = view.source("second", .{.x}); + const inputs = [_]Source.Placed{ assembled.withPartitioning(.replicated, .{}), second.withPartitioning(.replicated, .{}) }; + const tensor = Source.bindProgram(&inputs, second.shape(), Sub{}); + const compute = fixture.store.bindingOf(tensor.id).?.compute; + try std.testing.expectEqual(2, compute.inputs.len); + try std.testing.expect(compute.program.as(Sub) != null and compute.program.as(LoaderTestFixture.Identity) == null); + try std.testing.expect(compute.inputs[0].whole() == null); + try std.testing.expect(compute.inputs[1].whole() != null); + var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + var output: Buffer = undefined; + try loader.start(&tensor, &output, &fixture.store, .none); + try loader.wait(); + defer output.deinit(); + // { 1, 2, 7, 8 } - { 5, 6, 7, 8 }, wrapping. + try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); +} + +test "a padded host-mapped table stays in host memory and serves device gathers in place" { + // The engram arrangement: one source of 264-byte rows, an odd row count + // padded to a multiple of 8, rows sharded over every device, delivered + // as device-kind views of pinned host memory. A program gathers rows + // from each device's local shard. + try expectHostTableGathers(.rows); +} + +test "a replicated host-mapped table is one allocation that every device reads in place" { + // The same table on every device: the direct loader makes it one + // allocation (`direct_loader.DestinationState.initHost`), each device's + // shard a view of it, and every device's kernels gather from it. + try expectHostTableGathers(.replicated); +} + +/// Loads a padded table as `host_mapped`, rows sharded over every device or +/// replicated on each, and gathers rows of each device's shard on that +/// device. The table is freed while the gather is queued behind matrix +/// products: its memory must outlive the read, and go once it is done. +fn expectHostTableGathers(placement: enum { rows, replicated }) !void { + const allocator = std.testing.allocator; + const io = std.testing.io; + const rows: usize = 100_003; + const row_bytes: usize = 264; + const gathered: usize = 256; + + // The accelerator under test, or four CPU devices. + const shared = testing.env(); + const owned: ?*Platform = if (shared.target == .cpu) + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 4 } }) + else + null; + defer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const api = platform.pjrt_api; + const mesh: *const Sharding.Mesh = if (owned) |p| + try p.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) + else + shared.meshes.get("model").?; + if (!backend.supportsMemory(platform, .host_mapped)) return error.SkipZigTest; + const devices = mesh.devicesInCanonicalOrder(); + const shard_count = devices.len; + + const padded_rows = std.mem.alignForward(usize, rows, 8); + const shard_rows = if (placement == .rows) padded_rows / shard_count else padded_rows; + if (placement == .rows) try std.testing.expectEqual(padded_rows, shard_rows * shard_count); + const padded = try allocator.alloc(u8, padded_rows * row_bytes); + defer allocator.free(padded); + @memset(padded, 0); + for (0..rows) |row| { + const bytes = padded[row * row_bytes ..][0..row_bytes]; + // The row's number first, so a misplaced row cannot pass for another. + std.mem.writeInt(u32, bytes[0..4], @intCast(row + 1), .little); + for (bytes[4..], 4..) |*byte, column| byte.* = @as(u8, @truncate(row *% 131 +% column *% 7)) | 1; + } + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try testing.writeSafetensors(allocator, io, tmp.dir, "table.safetensors", ®istry, &.{ + .{ .name = "table", .dtype = "U8", .dims = &.{ @intCast(rows), @intCast(row_bytes) }, .bytes = padded[0 .. rows * row_bytes] }, + }); + var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{mesh} }); + defer store.deinit(); + const source = store.view().source("table", .{ .row, .channel }) + .pad(0, .{ .row = TensorStore.Source.Pad{ .high = @intCast(padded_rows - rows) } }); + const table = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + store.view().setMemory(table, .host_mapped); + try std.testing.expectEqual(@as(i64, @intCast(padded_rows)), table.dim(.row)); + + const live_before = direct_loader.liveHostAllocations(); + var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 4 }); + defer loader.deinit(); + // The device allocators' counts, once the set-up's calibration has + // allocated on the devices. Bytes in use would not do: CUDA's counts + // what its pool holds, which freed memory leaves room in. CPU's plugin + // counts nothing. + try loader.awaitSetup(); + var allocations: [Platform.MAX_NUM_DEVICES]?u64 = undefined; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_table: Buffer = undefined; + try loader.start(&table, &device_table, &store, .none); + try loader.wait(); + var freed = false; + defer if (!freed) device_table.deinit(); + // The table took no device memory: nothing was allocated on a device. + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expectEqual(before, platform.devices[device.id].memoryStats().num_allocs.?); + }; + try std.testing.expectEqual(padded.len, loader.stats.bytes_loaded); + try testing.expectShards(allocator, io, &device_table, padded, .host_mapped); + // A row shard is one allocation per device, a replicated table one. + try std.testing.expectEqual(live_before + if (placement == .rows) shard_count else 1, direct_loader.liveHostAllocations()); + + const first_pointer = try device_table._shards.get(0).opaqueDeviceMemoryDataPointer(api); + for (devices, device_table._shards.constSlice()) |device, shard| { + // Replicas share one allocation; row shards never do. + const pointer = try shard.opaqueDeviceMemoryDataPointer(api); + const shares = placement == .replicated and shard_count > 1; + if (device.id != devices[0].id) try std.testing.expectEqual(shares, pointer == first_pointer); + } + + // Each device gathers `gathered` rows of its own shard, the padding + // rows of the last one included (of every replica, replicated), once + // a chain of matrix products is done: the indices depend on them. + const Gather = struct { + fn forward(table_: Tensor, indices: Tensor, ballast: Tensor) Tensor { + var product = ballast; + for (0..8) |_| product = product.dot(product.withTags(.{ .k, .j }), .k).withTags(.{ .i, .k }); + // Zero, as every product keeps the ballast's positive entries. + const offset: Tensor = .select( + product.sum(.i).sum(.k).reshape(Shape.init(.{}, .f32)).cmp(.LT, .scalar(0, .f32)), + .scalar(1, .u32), + .scalar(0, .u32), + ); + const output: Shape = Shape.init(.{ .shard = indices.dim(.shard), .bytes = indices.dim(.k) * table_.dim(.channel) }, .u8) + .withPartitioning(table_.shape()._sharding.mesh.?, .{ .shard = .model }); + return ops.manualComputation((struct { + fn body(inputs: struct { table: Tensor, indices: Tensor, offset: Tensor }, local: Shape) Tensor { + const shifted = inputs.indices.add(inputs.offset.broad(inputs.indices.shape())); + return inputs.table.gather(.{ .row = shifted }, .{}).reshape(local); + } + }).body, .{ .table = table_, .indices = indices, .offset = offset }, output); + } + }; + const indices_shape = Shape.init(.{ .shard = shard_count, .k = gathered }, .u32).withPartitioning(mesh, .{ .shard = .model }); + const indices = try allocator.alloc(u32, shard_count * gathered); + defer allocator.free(indices); + for (0..shard_count) |shard| { + const local = indices[shard * gathered ..][0..gathered]; + for (local, 0..) |*index, k| index.* = @intCast((k * 7919 + shard * 104_729) % shard_rows); + // The first and last rows of every shard; on the last device these + // are padding. + local[0] = 0; + local[1] = @intCast(shard_rows - 1); + local[2] = @intCast(shard_rows - 2); + } + var indices_buffer: Buffer = try .from(io, platform, indices_shape, std.mem.sliceAsBytes(indices), .{}); + defer indices_buffer.deinit(); + // Milliseconds of products on an accelerator; a token amount on CPU. + const ballast_dim: usize = if (platform.target == .cpu) 32 else 4096; + const ballast_shape = Shape.init(.{ .i = ballast_dim, .k = ballast_dim }, .f32).withPartitioning(mesh, .{}); + const ballast = try allocator.alloc(f32, ballast_dim * ballast_dim); + defer allocator.free(ballast); + @memset(ballast, 1.0 / @as(f32, @floatFromInt(ballast_dim))); + var ballast_buffer: Buffer = try .from(io, platform, ballast_shape, std.mem.sliceAsBytes(ballast), .{}); + defer ballast_buffer.deinit(); + + const exe = try platform.compileFn(allocator, io, Gather.forward, .{ Tensor.fromShape(table.shape()), Tensor.fromShape(indices_shape), Tensor.fromShape(ballast_shape) }, .{ .meshes = &.{mesh} }); + defer exe.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + args.set(.{ device_table, indices_buffer, ballast_buffer }); + try exe.call(io, args, &results, .{}); + var output = results.get(Buffer); + defer output.deinit(); + + // The table goes while the gather may still have to read it. Pinned + // buffers of its size and another pattern would take its memory had it + // gone early, and the gather would return their bytes. + device_table.deinit(); + freed = true; + const decoy_bytes = try allocator.alloc(u8, padded.len); + defer allocator.free(decoy_bytes); + @memset(decoy_bytes, 0xee); + var decoy: Buffer = try .from(io, platform, table.shape(), decoy_bytes, .{ .memory = .host_pinned }); + defer decoy.deinit(); + + const expected = try allocator.alloc(u8, shard_count * gathered * row_bytes); + defer allocator.free(expected); + for (0..shard_count) |shard| { + for (0..gathered) |k| { + const first_row = if (placement == .rows) shard * shard_rows else 0; + const row = first_row + indices[shard * gathered + k]; + @memcpy(expected[(shard * gathered + k) * row_bytes ..][0..row_bytes], padded[row * row_bytes ..][0..row_bytes]); + } + } + try testing.expectShards(allocator, io, &output, expected, .default); + + // The allocations go with their last views, once nothing reads them; + // PJRT may drop the views on its own threads just after the output is + // ready. + var polls: usize = 0; + while (direct_loader.liveHostAllocations() != live_before) : (polls += 1) { + if (polls == 5000) return error.TestUnexpectedResult; + try io.sleep(.fromMilliseconds(1), .awake); + } + + // The control: the same table in device memory is allocated on every + // device, so the check after the load above can fail. + const device_copy = switch (placement) { + .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), + .replicated => source.withPartitioning(.model, .{}).bind(), + }; + for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; + var device_buffer: Buffer = undefined; + try loader.start(&device_copy, &device_buffer, &store, .none); + try loader.wait(); + defer device_buffer.deinit(); + for (devices) |device| if (allocations[device.id]) |before| { + try std.testing.expect(platform.devices[device.id].memoryStats().num_allocs.? > before); + }; +} + +/// A loader on the buffered backend, whichever backend `platform` loads +/// through (`backend.loadsDirect`), its set-up done in place: CPU loads +/// through the direct backend, and this keeps the front end over the +/// buffered one under test. +fn initBuffered(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Loader.Options) !Loader { + const backend_opts = try backendOptions(io, opts); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .setup = null, + .backend = .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, backend_opts) }, + .program_group_bytes = opts.program_group_bytes, + }; +} + +test "the front end loads through the buffered backend and fails with it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var fixture: LoaderTestFixture = undefined; + try fixture.init(allocator, io); + defer fixture.deinit(); + var loader = try initBuffered(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); + defer loader.deinit(); + + // A tensor, one assembled from value's first half and second's last + // half, one padded with zeros, and a program tensor, in a group of its + // own. + const Source = TensorStore.Source; + const view = fixture.store.view(); + const Model = struct { value: Tensor, assembled: Tensor, padded: Tensor, computed: Tensor }; + const model: Model = .{ + .value = fixture.value, + .assembled = Source.concatenate(&.{ + view.source("value", .{.x}).slice(.x, .{ .end = 2 }), + view.source("second", .{.x}).slice(.x, .{ .start = 2 }), + }, .x).withPartitioning(.replicated, .{}).bind(), + .padded = view.source("second", .{.x}).pad(0, .{ .x = Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), + .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Add{ .value = 1 }), + }; + var buffers = try mem.bufferize(allocator, &model); + defer mem.deinitBufferized(allocator, &buffers); + try loader.start(&model, &buffers, &fixture.store, .none); + errdefer loader.cancel(); + try loader.wait(); + try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); + try testing.expectShards(allocator, io, &buffers.assembled, &.{ 1, 2, 7, 8 }, .default); + try testing.expectShards(allocator, io, &buffers.padded, &.{ 5, 6, 7, 8, 0, 0 }, .default); + try testing.expectShards(allocator, io, &buffers.computed, &.{ 2, 3, 4, 5 }, .default); + + // A missing file fails the loader, and every later call with it. + var broken: Buffer = undefined; + try loader.start(&fixture.missing, &broken, &fixture.store, .none); + defer broken.deinit(); + try std.testing.expectError(error.FileNotFound, loader.wait()); + try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); + try std.testing.expectEqual(0, broken._shards.len); + try std.testing.expectError(error.FileNotFound, loader.start(&fixture.value, &broken, &fixture.store, .none)); +} diff --git a/zml/io/log.zig b/zml/io/log.zig new file mode 100644 index 00000000..e5bb5820 --- /dev/null +++ b/zml/io/log.zig @@ -0,0 +1,38 @@ +//! The loader's log scopes. A failure is logged once, where it is +//! detected, at `err` with the values that explain it; cancellation and a +//! plain allocation failure log at `debug`. Frames that only propagate an +//! error do not log it again. Test builds log `err` at `debug`, since the +//! test runner counts logged errors as failures: error-path tests assert +//! the returned errors and the cleanup, not the logs. +const std = @import("std"); +const builtin = @import("builtin"); + +pub const load = Scoped(.@"zml/io/load"); +pub const io = Scoped(.@"zml/io"); +pub const mem = Scoped(.@"zml/mem"); + +fn Scoped(comptime scope: @EnumLiteral()) type { + const base = std.log.scoped(scope); + return struct { + pub const debug = base.debug; + pub const info = base.info; + pub const warn = base.warn; + + /// A terminal failure. The test runner counts logged errors as + /// test failures, so test builds log it at `debug` instead. + pub fn err(comptime fmt: []const u8, args: anytype) void { + if (builtin.is_test) base.debug(fmt, args) else base.err(fmt, args); + } + + /// The first failure of a pipeline, with the error's name appended. + /// Cancellation is expected, and an `OutOfMemory` is a plain + /// allocation failure, returned unlogged, or was logged where it + /// failed (host memory): those log at `debug`. + pub fn failure(e: anyerror, comptime fmt: []const u8, args: anytype) void { + const named_fmt = fmt ++ ": {s}"; + const named_args = args ++ .{@errorName(e)}; + const logged = e == error.Canceled or e == error.Cancelled or e == error.OutOfMemory; + if (logged) debug(named_fmt, named_args) else err(named_fmt, named_args); + } + }; +} diff --git a/zml/kernel.zig b/zml/kernel.zig new file mode 100644 index 00000000..03a64d70 --- /dev/null +++ b/zml/kernel.zig @@ -0,0 +1,1614 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +const cute_builder = @import("dsl/cute/builder"); +const fly_builder = @import("dsl/fly/builder"); +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +const triton_builder = @import("dsl/triton/builder"); + +const Compiler = @import("Compiler.zig"); +const DataType = @import("dtype.zig").DataType; +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const platform_ = @import("platform.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub const triton = struct { + pub const Builder = triton_builder.Builder; + pub const Value = triton_builder.Value; + pub const DType = triton_builder.DType; + pub const FinishError = triton_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; + } +}; + +pub const mosaic_tpu = struct { + const Builder = mosaic_tpu_builder.Builder; + const DType = mosaic_tpu_builder.DType; + const FinishError = mosaic_tpu_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; + } + + const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + }; + + const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, + }; + + const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, + }; + + const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + }; + + fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, + ) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); + } + + const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, + }; + + fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); + } +}; + +pub const cuda_tile = struct { + pub const Builder = cuda_tile_builder.Builder; + pub const Value = cuda_tile_builder.Value; + pub const DType = cuda_tile_builder.DType; + pub const FinishError = cuda_tile_builder.FinishError; + pub const EntryHint = cuda_tile_builder.EntryHint; + pub const Arch = cuda_tile_builder.Arch; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const cute = struct { + pub const Builder = cute_builder.Builder; + pub const ArgSpec = cute_builder.ArgSpec; + pub const Value = cute_builder.Value; + pub const View = cute_builder.View; + pub const Atom = cute_builder.Atom; + pub const DType = cute_builder.DType; + pub const AlgebraToken = cute_builder.AlgebraToken; + pub const LayoutSpec = cute_builder.LayoutSpec; + pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; + pub const FinishError = cute_builder.FinishError; + pub const TmaConfig = cute_builder.TmaConfig; + pub const TmaFormat = cute_builder.TmaFormat; + pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } + + fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A complete CuTe program: one or more `cuda.kernel` operations inside + /// a `gpu.module`, followed by a public `func.func` which constructs the + /// launch-time CuTe objects and calls `cuda.launch_ex`. + /// + /// This is the source-level equivalent of invoking a Python `@cute.jit` + /// object whose `__call__` builds TMA descriptors and launches an + /// `@cute.kernel`. XLA executes the generated host function on its CUDA + /// stream, so the custom call deliberately carries no external grid or + /// block dimensions. + pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const tcl = struct { + pub const Builder = tcl_builder.Builder; + pub const TensorOperation = tcl_builder.TensorOperation; + pub const Value = tcl_builder.Value; + pub const Axis = tcl_builder.Axis; + pub const Expr = tcl_builder.Expr; + pub const Dram = tcl_builder.Dram; + pub const Context = tcl_builder.Context; + pub const DType = tcl_builder.DType; + pub const VeOp = tcl_builder.VeOp; + pub const ReduceMode = tcl_builder.ReduceMode; + pub const Predicate = tcl_builder.Predicate; + pub const Tactic = tcl_builder.Tactic; + pub const FinishError = tcl_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A TCL function compiled by the Furiosa plugin as one native unit: + /// `declareArgs` declares the inputs in order, `ret` the outputs. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; + } +}; + +pub const metal = struct { + pub const Builder = metal_builder.Builder; + pub const Value = metal_builder.Value; + pub const Type = metal_builder.Type; + pub const Struct = metal_builder.Struct; + pub const ArgSpec = metal_builder.ArgSpec; + pub const LoopOpts = metal_builder.LoopOpts; + pub const FinishError = metal_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); + } + + pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; + } + + /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host + /// `extern struct` laid out like a `constant S &` argument. + pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } + }; + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; + } + + /// A Metal kernel: `run` declares its arguments with `declareArgs`, by + /// name, and builds its body. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const fly = struct { + /// Threads per wavefront: 64 on CDNA, 32 on RDNA. + pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; + } + + /// The plugin sizes a block in wavefronts, so a thread count has to divide + /// evenly: a kernel's layouts are built for an exact number of threads and + /// spare ones would partition out of range. + fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); + } + + pub const Builder = fly_builder.Builder; + pub const Value = fly_builder.Value; + pub const DType = fly_builder.DType; + pub const FinishError = fly_builder.FinishError; + pub const TiledCopy = fly_builder.TiledCopy; + pub const TiledMma = fly_builder.TiledMma; + pub const Arch = fly_builder.Arch; + pub const MmaFlavor = fly_builder.MmaFlavor; + pub const fly_rocdl = fly_builder.fly_rocdl; + + /// The matrix atom this device provides, or null when it has none: WMMA + /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. + pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; + } + pub const layout = fly_builder.layout; + pub const Layout = fly_builder.Layout; + pub const Tile = fly_builder.Tile; + pub const IntTuple = fly_builder.IntTuple; + pub const L = fly_builder.L; + pub const rowMajor = fly_builder.rowMajor; + pub const colMajor = fly_builder.colMajor; + pub const ordered = fly_builder.ordered; + pub const tile = fly_builder.tile; + pub const it = fly_builder.it; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernels run on rocm" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: Tensor) Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const model = platform.meshes.get("model").?; + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(model, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: Tensor) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + return .{ r[0], r[1] }; + } + }; + const x: Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{model} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: Tensor, y: Tensor) Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel deleted file mode 100644 index 61ab44f8..00000000 --- a/zml/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/kernel/cuda_tile", - "//zml/kernel/cute", - "//zml/kernel/fly", - "//zml/kernel/metal", - "//zml/kernel/mosaic_tpu", - "//zml/kernel/tcl", - "//zml/kernel/triton", - ], -) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel deleted file mode 100644 index 89b7977e..00000000 --- a/zml/kernel/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cuda_tile", - srcs = [ - "cuda_tile.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig deleted file mode 100644 index 3f6d8734..00000000 --- a/zml/kernel/cuda_tile/cuda_tile.zig +++ /dev/null @@ -1,163 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -pub const Builder = cuda_tile_builder.Builder; -pub const Value = cuda_tile_builder.Value; -pub const DType = cuda_tile_builder.DType; -pub const FinishError = cuda_tile_builder.FinishError; -pub const EntryHint = cuda_tile_builder.EntryHint; -pub const Arch = cuda_tile_builder.Arch; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig deleted file mode 100644 index c929b0b9..00000000 --- a/zml/kernel/cuda_tile/test.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -const cuda_tile = @import("cuda_tile.zig"); - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel deleted file mode 100644 index 7a613072..00000000 --- a/zml/kernel/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "cute.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig deleted file mode 100644 index 2e8c919f..00000000 --- a/zml/kernel/cute/cute.zig +++ /dev/null @@ -1,264 +0,0 @@ -const std = @import("std"); - -const cute_builder = @import("dsl/cute/builder"); -pub const AlgebraExpr = cute_builder.AlgebraExpr; -pub const Builder = cute_builder.Builder; -pub const ArgSpec = cute_builder.ArgSpec; -pub const Value = cute_builder.Value; -pub const View = cute_builder.View; -pub const Atom = cute_builder.Atom; -pub const DType = cute_builder.DType; -pub const AlgebraToken = cute_builder.AlgebraToken; -pub const LayoutSpec = cute_builder.LayoutSpec; -pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; -pub const FinishError = cute_builder.FinishError; -pub const TmaConfig = cute_builder.TmaConfig; -pub const TmaFormat = cute_builder.TmaFormat; -pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; -pub const CtaGroup = cute_builder.CtaGroup; -pub const ctaCount = cute_builder.ctaCount; -pub const MBarrierScope = cute_builder.MBarrierScope; -pub const OperandMajorMode = cute_builder.OperandMajorMode; -pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; -pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; -pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; -pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; -pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; -pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} - -fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A complete CuTe program: one or more `cuda.kernel` operations inside -/// a `gpu.module`, followed by a public `func.func` which constructs the -/// launch-time CuTe objects and calls `cuda.launch_ex`. -/// -/// This is the source-level equivalent of invoking a Python `@cute.jit` -/// object whose `__call__` builds TMA descriptors and launches an -/// `@cute.kernel`. XLA executes the generated host function on its CUDA -/// stream, so the custom call deliberately carries no external grid or -/// block dimensions. -pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig deleted file mode 100644 index f4d457a3..00000000 --- a/zml/kernel/cute/test.zig +++ /dev/null @@ -1,34 +0,0 @@ -const std = @import("std"); - -const cute = @import("cute.zig"); - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel deleted file mode 100644 index 57203521..00000000 --- a/zml/kernel/fly/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "fly", - srcs = [ - "fly.zig", - "test.zig", - ], -) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig deleted file mode 100644 index b7d6c943..00000000 --- a/zml/kernel/fly/fly.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); -pub const Builder = fly_builder.Builder; -pub const Value = fly_builder.Value; -pub const DType = fly_builder.DType; -pub const FinishError = fly_builder.FinishError; -pub const TiledCopy = fly_builder.TiledCopy; -pub const TiledMma = fly_builder.TiledMma; -pub const Arch = fly_builder.Arch; -pub const MmaFlavor = fly_builder.MmaFlavor; -pub const fly_rocdl = fly_builder.fly_rocdl; -pub const layout = fly_builder.layout; -pub const Layout = fly_builder.Layout; -pub const Tile = fly_builder.Tile; -pub const IntTuple = fly_builder.IntTuple; -pub const L = fly_builder.L; -pub const rowMajor = fly_builder.rowMajor; -pub const colMajor = fly_builder.colMajor; -pub const ordered = fly_builder.ordered; -pub const tile = fly_builder.tile; -pub const it = fly_builder.it; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -/// Threads per wavefront: 64 on CDNA, 32 on RDNA. -pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; -} - -/// The plugin sizes a block in wavefronts, so a thread count has to divide -/// evenly: a kernel's layouts are built for an exact number of threads and -/// spare ones would partition out of range. -fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); -} - -/// The matrix atom this device provides, or null when it has none: WMMA -/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. -pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig deleted file mode 100644 index ae788d96..00000000 --- a/zml/kernel/fly/test.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); - -const fly = @import("fly.zig"); -const zml = @import("../../zml.zig"); - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -test "fly kernels run on rocm" { - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig deleted file mode 100644 index 204c6204..00000000 --- a/zml/kernel/kernel.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); -pub const cute = @import("cute/cute.zig"); -pub const fly = @import("fly/fly.zig"); -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); - -test { - const std = @import("std"); - std.testing.refAllDecls(@This()); -} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel deleted file mode 100644 index 4007beb4..00000000 --- a/zml/kernel/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - "test.zig", - ], -) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig deleted file mode 100644 index 9626a1de..00000000 --- a/zml/kernel/metal/metal.zig +++ /dev/null @@ -1,237 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -pub const Builder = metal_builder.Builder; -pub const Value = metal_builder.Value; -pub const Type = metal_builder.Type; -pub const Struct = metal_builder.Struct; -pub const ArgSpec = metal_builder.ArgSpec; -pub const LoopOpts = metal_builder.LoopOpts; -pub const FinishError = metal_builder.FinishError; -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); -} - -pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; -} - -/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host -/// `extern struct` laid out like a `constant S &` argument. -pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } -}; - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; -} - -/// A Metal kernel: `run` declares its arguments with `declareArgs`, by -/// name, and builds its body. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig deleted file mode 100644 index 0a342af0..00000000 --- a/zml/kernel/metal/test.zig +++ /dev/null @@ -1,71 +0,0 @@ -const std = @import("std"); - -const metal = @import("metal.zig"); -const zml = @import("../../zml.zig"); - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: zml.Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel deleted file mode 100644 index ba3b6b66..00000000 --- a/zml/kernel/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = ["mosaic_tpu.zig"], -) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index a09e8368..00000000 --- a/zml/kernel/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,284 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const Builder = mosaic_tpu_builder.Builder; -const DType = mosaic_tpu_builder.DType; -const FinishError = mosaic_tpu_builder.FinishError; -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; -} - -const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, -}; - -const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, -}; - -const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, -}; - -const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, -}; - -fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, -) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); -} - -const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, -}; - -fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); -} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel deleted file mode 100644 index bc7a0aa2..00000000 --- a/zml/kernel/tcl/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "tcl", - srcs = [ - "tcl.zig", - "test.zig", - ], -) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig deleted file mode 100644 index 023e862f..00000000 --- a/zml/kernel/tcl/tcl.zig +++ /dev/null @@ -1,206 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -pub const Builder = tcl_builder.Builder; -pub const TensorOperation = tcl_builder.TensorOperation; -pub const Value = tcl_builder.Value; -pub const Axis = tcl_builder.Axis; -pub const Expr = tcl_builder.Expr; -pub const Dram = tcl_builder.Dram; -pub const Context = tcl_builder.Context; -pub const DType = tcl_builder.DType; -pub const VeOp = tcl_builder.VeOp; -pub const ReduceMode = tcl_builder.ReduceMode; -pub const Predicate = tcl_builder.Predicate; -pub const Tactic = tcl_builder.Tactic; -pub const FinishError = tcl_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A TCL function compiled by the Furiosa plugin as one native unit: -/// `declareArgs` declares the inputs in order, `ret` the outputs. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; -} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig deleted file mode 100644 index b9ae1875..00000000 --- a/zml/kernel/tcl/test.zig +++ /dev/null @@ -1,123 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const tcl = @import("tcl.zig"); - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: zml.Tensor) zml.Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: zml.Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel deleted file mode 100644 index 1e992352..00000000 --- a/zml/kernel/triton/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "triton", - srcs = ["triton.zig"], -) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig deleted file mode 100644 index 6a91a03f..00000000 --- a/zml/kernel/triton/triton.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const triton_builder = @import("dsl/triton/builder"); -pub const Builder = triton_builder.Builder; -pub const Value = triton_builder.Value; -pub const DType = triton_builder.DType; -pub const FinishError = triton_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const ops = @import("../../ops.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; -} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index 81a35cf4..b5fe7682 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [tests.zig](tests.zig) compare results +The tests in [flash_attention.zig](flash_attention.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index ec0fce16..0d01c357 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -1,7 +1,4 @@ const std = @import("std"); -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); -const vector = @import("mlir/dialects/vector"); const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; @@ -17,8 +14,6 @@ pub const Config = struct { head_dim: i64, dtype: cute.DType, splits: i64 = 1, - /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. - causal: bool = true, }; /// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA @@ -149,16 +144,13 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void const query = row.div(group).minimum(cfg.query_length - 1); const head = kv_head.mul(group).add(row.rem(group)); const start = a.positions.get(.{batch}); - const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); - const live_end = if (cfg.causal) - start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) - else - b.constant(.i32, cfg.key_length); + const position = start.add(query); + const causal_end = start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1); // Partition the live cache, rather than its capacity, so decode uses all // blocks even when only a prefix of the cache is populated. - const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); + const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); - const end = if (split) begin.add(chunk).minimum(live_end) else live_end; + const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); @@ -313,494 +305,6 @@ fn merge(b: *B, cfg: Config) cute.FinishError!void { } } -// ==================== Prefill ==================== -// -// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 -// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one -// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and -// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, -// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied -// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. -// -// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared -// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas -// caps at 48 KiB before sm_120. - -pub const Prefill = cute.Program(Config, .{ - .name = "flash_attention_cute_prefill", - .inputs = &.{ "q", "k", "v", "positions" }, - .outputs = &.{"out"}, - .run = prefillProgram, -}); - -const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; - -/// Keys per tile (FA2's kBlockN). -const prefill_keys = 64; - -/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. -/// Same as FA2 we only support 16-bit inputs -pub fn prefillSupported(cfg: Config) bool { - const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; - const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; - return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; -} - -/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. -fn prefillRowTiles(head_dim: i64) i64 { - return if (head_dim <= 128) 2 else 1; -} - -/// Queries per block (FA2's kBlockM). -fn prefillRows(cfg: Config) i64 { - return 4 * 16 * prefillRowTiles(cfg.head_dim); -} - -/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. -fn prefillGrid(cfg: Config) [3]i32 { - return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; -} - -fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { - std.debug.assert(prefillSupported(cfg)); - switch (cfg.head_dim) { - inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { - inline .bf16, .f16 => |dt| { - b.beginFunction(prefill_kernel_name, .cuda_kernel); - try prefillSized(b, cfg, d, dt); - b.endFunction(block); - - b.beginFunction(Prefill.name, .host); - const host = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .positions = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = dt }, - }); - const grid = prefillGrid(cfg); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, - .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, - .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), - .stream = b.cudaStream(), - }); - const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); - b.returnHostStatus(b.cudaResultStatus(launch)); - b.endFunction(null); - }, - else => unreachable, - }, - else => unreachable, - } -} - -/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and -/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. -fn fmax(b: *B, x: Value, y: Value) Value { - return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; -} - -fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { - var result: [n]Value = undefined; - for (&result, 0..) |*r, i| { - r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); - } - return result; -} - -/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. -fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { - const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ - .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), - .constraints = "=r,=r,=r,=r,r", - .has_side_effects = true, - }, b.loc())); - return unpackStruct(b, 4, result, .i32); -} - -/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). -fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { - const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ - "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ - .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), - .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", - }, b.loc())); - return unpackStruct(b, 4, result, .f32); -} - -/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop -/// carries at most 64 values). -fn packAcc(b: *B, values: [4]Value) Value { - return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); -} - -fn unpackAcc(b: *B, value: Value) [4]Value { - var result: [4]Value = undefined; - for (&result, 0..) |*r, i| { - r.* = b.vectorExtract(value, @intCast(i), .f32); - } - return result; -} - -/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. -/// -/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so -/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses -/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a -/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from -/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant -/// (XORed into the 16-byte vector index). -fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { - return struct { - const row_tiles = prefillRowTiles(d); - const rows = 4 * 16 * row_tiles; - const n = prefill_keys; - const score_tiles = n / 8; - const out_tiles = d / 8; - const row_bytes = d * 2; - /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. - const swizzle_rows = @min(8, d / 8); - const q_bytes = rows * row_bytes; - const kv_bytes = n * row_bytes; - /// Rows covered by one 16-byte cp.async per thread of the block. - const copy_rows = @divExact(128, d / 8); - /// Running max and per-thread partial sum of the two rows of every row tile. - const stats = 2 * row_tiles; - /// Carried by the key loop: maxima, sums, then the packed output accumulators of every - /// (row tile, 8 output features). - const carried = 2 * stats + row_tiles * out_tiles; - const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); - - comptime { - // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. - std.debug.assert(std.math.isPowerOfTwo(copy_rows)); - std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); - } - - b: *B, - cfg: Config, - lane: Value, - warp: Value, - first_query: Value, - start: Value, - q_smem: Value, - k_smem: Value, - v_smem: Value, - k_ptr: Value, - v_ptr: Value, - kv_base: Value, - tiles: Value, - /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. - copy_row: Value, - copy_col: Value, - /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. - copy_row_offset: Value, - copy_vector_offset: Value, - /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. - q_frag_row: Value, - k_frag_row: Value, - v_frag_row: Value, - qv_frag_vector: Value, - k_frag_vector: Value, - - const Self = @This(); - - fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { - const lane = tid.rem(32); - const warp = tid.div(32); - const copy_row = tid.div(d / 8); - const copy_col = tid.rem(d / 8); - const lane8 = lane.rem(8); - // The fragment rows a thread reads are lane % 8 plus multiples of 8. - const lane_xor = lane.rem(swizzle_rows); - return .{ - .b = b, - .cfg = cfg, - .lane = lane, - .warp = warp, - .first_query = first_query, - .start = start, - .q_smem = shared, - .k_smem = shared.add(q_bytes), - .v_smem = shared.add(q_bytes + kv_bytes), - .k_ptr = k_ptr, - .v_ptr = v_ptr, - .kv_base = kv_base, - .tiles = tiles, - .copy_row = copy_row, - .copy_col = copy_col, - .copy_row_offset = copy_row.mul(row_bytes), - .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), - // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). - .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), - // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). - .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), - // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). - .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), - .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), - .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), - }; - } - - /// Destination of the i-th cp.async of this thread in a tile starting at `base`. - fn copyDst(t: Self, base: Value, comptime i: usize) Value { - const xor = ((copy_rows * i) % swizzle_rows) * 16; - const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); - return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); - } - - /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. - fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { - const b = t.b; - const stride = t.cfg.kv_heads * d; - const first_key = key_base.add(t.copy_row); - const safe = src.add(t.kv_base.mul(2)); - const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); - inline for (0..@divExact(n, copy_rows)) |i| { - const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); - const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); - b.cpAsync16(t.copyDst(dst, i), source, valid); - } - } - - fn queryOf(t: Self, mi: usize, h: usize) Value { - return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); - } - - fn accIndex(comptime mi: usize, comptime di: usize) usize { - return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); - } - - /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are - /// fully visible to every query of the block. - fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { - const b = t.b; - const lane = t.lane; - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const key_base = tile.mul(n); - b.cpAsyncWaitAll(); - b.syncThreads(); - t.loadTile(t.v_ptr, t.v_smem, key_base); - - // S = Q K^T, k-steps of 16 features. - var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); - inline for (0..d / 16) |ks| { - var qa: [row_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); - } - const k_vector = t.k_frag_vector.bitXor(ks * 32); - inline for (0..score_tiles / 2) |np| { - const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); - inline for (0..row_tiles) |mi| { - s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); - s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); - } - } - } - - // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: - // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). - var next: [carried]Value = undefined; - var alpha: [row_tiles][2]Value = undefined; - inline for (0..row_tiles) |mi| { - if (masked) { - inline for (0..score_tiles) |ni| { - inline for (0..4) |j| { - const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); - const in_range = key.lt(t.cfg.key_length); - const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; - s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); - } - } - } - inline for (0..2) |h| { - var row_max = s[mi][0][h * 2]; - inline for (0..score_tiles) |ni| { - inline for (0..2) |j| { - row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); - } - } - inline for (.{ 2, 1 }) |offset| { - row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); - } - const old_max = carried_in[mi * 2 + h]; - const new_max = fmax(b, old_max, row_max); - // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. - const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); - alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); - next[mi * 2 + h] = new_max; - var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); - inline for (0..score_tiles) |ni| { - inline for (0..2) |j| { - s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); - sum = sum.add(s[mi][ni][h * 2 + j]); - } - } - next[stats + mi * 2 + h] = sum; - } - } - - // All warps are done with K_j once V_j has landed: start copying K_{j+1}. - b.cpAsyncWaitAll(); - b.syncThreads(); - var prefetch = b.openIf(tile.add(1).lt(t.tiles)); - { - t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); - prefetch.close(); - } - - // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. - var o: [row_tiles][out_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - inline for (0..out_tiles) |di| { - const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); - inline for (0..4) |j| { - o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); - } - } - } - inline for (0..n / 16) |ks| { - var pa: [row_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - pa[mi] = .{ - b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), - b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), - b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), - b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), - }; - } - const v_row = t.v_frag_row.add(ks * 16 * row_bytes); - inline for (0..out_tiles / 2) |dp| { - const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); - inline for (0..row_tiles) |mi| { - o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); - o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); - } - } - } - inline for (0..row_tiles) |mi| { - inline for (0..out_tiles) |di| { - next[accIndex(mi, di)] = packAcc(b, o[mi][di]); - } - } - return next; - } - }; -} - -fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { - const T = PrefillTile(d, dt); - const a = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .positions = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = dt }, - }); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); - - const tid = b.threadIdx().x; - const first_query = b.blockIdx().x.mul(T.rows); - const head = b.blockIdx().y; - const batch = b.blockIdx().z; - const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); - // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the - // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and - // `allocSmemView` both become static shared memory instead. - b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); - const fields = [_][]const u8{ - std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), - std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), - std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), - }; - const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); - const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); - const q_ptr = b.ptrToInt(a.q, .i64); - const out_ptr = b.ptrToInt(a.out, .i64); - const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; - - // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. - const q_stride = cfg.query_heads * d; - const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); - const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); - - // Causal: the last query of the block sees keys up to start + last_query, and every query of the - // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the - // keys, is masked. - const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); - const tiles = if (cfg.causal) - start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) - else - b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); - const full_tiles = if (cfg.causal) - start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) - else - b.constant(.i32, @divTrunc(cfg.key_length, T.n)); - - const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); - - // Out of range rows are zero-filled and never read from global memory. - { - const first_row = first_query.add(t.copy_row); - const safe = q_ptr.add(q_base.mul(2)); - const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); - inline for (0..@divExact(T.rows, T.copy_rows)) |i| { - const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); - b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); - } - } - t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); - - var initial: [T.carried]Value = undefined; - const zero = b.constant(.f32, 0); - for (initial[0..T.stats]) |*v| { - v.* = b.constant(.f32, -std.math.inf(f32)); - } - for (initial[T.stats .. 2 * T.stats]) |*v| { - v.* = zero; - } - for (initial[2 * T.stats ..]) |*v| { - v.* = packAcc(b, .{ zero, zero, zero, zero }); - } - - var full = b.openFor(0, full_tiles, 1, initial); - { - full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); - } - var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); - { - diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); - } - const results = diagonal.results[0..T.carried].*; - - // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. - inline for (0..T.row_tiles) |mi| { - inline for (0..2) |h| { - var sum = results[T.stats + mi * 2 + h]; - inline for (.{ 2, 1 }) |offset| { - sum = sum.add(b.shuffleXor(sum, offset)); - } - const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); - const query = t.queryOf(mi, h); - var valid = b.openIf(query.lt(cfg.query_length)); - { - const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); - inline for (0..T.out_tiles) |di| { - const acc = unpackAcc(b, results[T.accIndex(mi, di)]); - const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); - b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); - } - valid.close(); - } - } - } -} - test "CuTe attention emits tiled online softmax through head dimension 512" { for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { const ir = try Kernel.emit(std.testing.allocator, .{ diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 36e0953b..2958888d 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -15,8 +15,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major >= 8; } -/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); const batched = q.shape().hasTag(.b) != null; const rank: usize = if (batched) 4 else 3; @@ -25,7 +24,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); - stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); + stdx.debug.assert(q.dim(.q) > 0 and q.dim(.q) <= k.dim(.k) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq <= Lk and matching K/V lengths", .{}); stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); if (batched) { stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); @@ -34,52 +33,65 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); } + const mesh = zml.Compiler.current().resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(mesh, .{ .h = .model }); + const k_sharded = k.withPartitioning(mesh, .{ .h = .model }); + const v_sharded = v.withPartitioning(mesh, .{ .h = .model }); const positions = if (batched) token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); - const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); - const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); - - const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); - - const cfg: kernel.Config = .{ - .batch = if (batched) q_.dim(.b) else 1, - .query_length = q_.dim(.q), - .key_length = k_.dim(.k), - .query_heads = q_.dim(.h), - .kv_heads = k_.dim(.h), - .head_dim = q_.dim(.hd), - .dtype = zml.kernel.cute.from(q_.dtype()), - .splits = splits, - .causal = causal, - }; - - const out = if (kernel.prefillSupported(cfg)) - kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out - else if (cfg.splits > 1) b: { - const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ - .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), - }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, - .block = kernel.block, - }).out; - break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, - .block = kernel.block, - }).out; - } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, - .block = kernel.block, - }).out; - - return out.transpose(q.shape()); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q_sharded.shape()); } +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + positions: zml.Tensor, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const batched = self.q.shape().hasTag(.b) != null; + const q = if (batched) self.q.transpose(.{ .b, .q, .h, .hd }) else self.q.transpose(.{ .q, .h, .hd }); + const k = if (batched) self.k.transpose(.{ .b, .k, .h, .hd }) else self.k.transpose(.{ .k, .h, .hd }); + const v = if (batched) self.v.transpose(.{ .b, .k, .h, .hd }) else self.v.transpose(.{ .k, .h, .hd }); + + const splits = kernel.decodeSplits(q.shape(), k.shape(), decodeResources(zml.Compiler.current().platform)); + + const cfg: kernel.Config = .{ + .batch = if (batched) q.dim(.b) else 1, + .query_length = q.dim(.q), + .key_length = k.dim(.k), + .query_heads = q.dim(.h), + .kv_heads = k.dim(.h), + .head_dim = q.dim(.hd), + .dtype = zml.kernel.cute.from(q.dtype()), + .splits = splits, + }; + + const out = if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, + .block = kernel.block, + }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; + + return out.transpose(output_shape); + } +}; + /// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. /// If there are no devices, returns 0 for both values. /// If any device does not have the required attributes, returns 0 for both values. diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 3fb898db..c1a27a36 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,11 +6,9 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); -const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -128,12 +126,239 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { - zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { return switch (parameters) { - .cute => cute.attention(q, k, v, token_index, opts.causal), - // The NKI and Metal kernels are causal only. - .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), - .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), + .cute => cute.attention(q, k, v, token_index), + .nki => nki.attention(q, k, v, token_index, parameters.nki), + .metal => metal.attention(q, k, v, token_index, metadata.metal), }; } + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, std.enums.values(Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, backends: []const Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + for (token_index_h) |index| { + // Check for out of bound reads + std.debug.assert(index + max_q_idx <= max_k_idx); + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, zml.attention.attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: Metadata = .init(.fromBackend(backend)); + const parameters: Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + attention, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index f82ade45..855bdcca 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -2,8 +2,5 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = [ - "flash_attn.zig", - "metal.zig", - ], + srcs = ["metal.zig"], ) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig deleted file mode 100644 index de4b68da..00000000 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ /dev/null @@ -1,732 +0,0 @@ -//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` -//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, -//! fa_tiled otherwise. - -const std = @import("std"); - -const metal = @import("../../../kernel/kernel.zig").metal; - -const ArgSpec = metal.ArgSpec; -const Builder = metal.Builder; -const Type = metal.Type; -const Value = metal.Value; - -const flt_max = std.math.floatMax(f32); -/// A masked score: below any real one yet finite, since fast math assumes -/// no infinities. Halved so subtracting a running maximum cannot overflow. -const masked_score = -flt_max / 2.0; -const max_half = 65504.0; -const simd_width = 32; - -/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. -pub const FaVec = struct { - pub const Cfg = struct { - head_dim: i32, - heads: i32, - splits: i32, - }; - - pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ - .{ .name = "num_heads", .type = .int }, - .{ .name = "n_kv", .type = .int }, - .{ .name = "k_len", .type = .int }, - .{ .name = "scale", .type = .float }, - } }; - - pub const HostArgs = extern struct { - num_heads: i32, - n_kv: i32, - k_len: i32, - scale: f32, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec", - .inputs = &.{ "q", "k", "v", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), - } - const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, - .q = char_in, - .k = char_in, - .v = char_in, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, - }); - impl(b, cfg, a); - } - - fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; - - const num_heads = a.args.field("num_heads"); - const n_kv = a.args.field("n_kv"); - const group = num_heads.div(n_kv); - const groups = group.div(HG); - const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); - - const kv_len = a.token_index.load(r).add(1); - const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const position_bytes = n_kv.to(.ulong).mul(D * 2); - const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); - const k0 = a.k.add(first); - const v0 = a.v.add(first); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); - } - - const S = b.local(.float, 0.0); - const M = b.local(.float, -flt_max / 2.0); - - const ic0 = b.local(.int, iwg); - var blocks = b.openLoop(.{}); - { - const ic = ic0.mul(C); - var past = b.openIf(ic.ge(kv_len)); - { - blocks.breakLoop(); - past.close(); - } - - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); - dots.close(); - } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } - qk.close(); - } - - const pos = ic.add(a.tiisg.to(.int)); - const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); - - const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); - const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); - S.assign(S.mul(ms).add(b.simdSum(vs))); - - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); - rescale.close(); - } - - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); - dims.close(); - } - pv.close(); - } - - blocks.next(); - ic0.assign(ic0.add(live)); - blocks.close(); - } - - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } - } - var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); - } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); - } - } - if (first_row) |*scope| scope.close(); - } -}; - -fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { - return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); -} - -/// fa_vec_reduce: merges the fa_vec split partials. -pub const FaVecReduce = struct { - pub const Cfg = struct { - splits: i32, - head_dim: i32, - }; - - pub const simdgroups = 8; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec_reduce", - .inputs = &.{ "htmp", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVec.Args }, - .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); - const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; - - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - const live = liveSplits(b, a.token_index.load(r).add(1), NWG); - - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); - { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); - } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); - } - } -}; - -/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past -/// `live` output zeros. -pub const FaTiled = struct { - pub const Cfg = struct { - dtype: Type, - head_size: i32, - bq: i32 = 32, - tile_kv: i32 = 32, - num_threads: i32 = 128, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_tiled", - .inputs = &.{ "q", "k", "v", "token_index", "live" }, - .outputs = &.{"out"}, - .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, - .run = run, - }); - - fn supported(cfg: Cfg) bool { - const tile = tileConfig(cfg.head_size) orelse return false; - return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and - cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; - } - - pub fn tileConfig(head_size: i64) ?[3]i32 { - return switch (head_size) { - 32, 64, 96, 128 => .{ 32, 32, 128 }, - 256 => .{ 16, 16, 64 }, - 512 => .{ 8, 8, 32 }, - else => null, - }; - } - - pub fn threadgroupMemory(cfg: Cfg) u64 { - const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); - return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); - } - - /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. - fn fragCoord(b: *Builder, lane_id: Value) Value { - const qid = lane_id.to(.short).div(4).to(.short); - const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); - const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); - return b.vec(.short2, .{ fn_, fm }); - } - - fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { - var x = v.rvalue(); - inline for (.{ 1, 8 }) |mask| { - const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); - x = switch (op) { - .max => b.max(x, y), - .sum => x.add(y), - }; - } - return x; - } - - fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { - p.store(i, p.load(i).insert(lane, x)); - } - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); - const T = cfg.dtype; - const HEAD_SIZE = cfg.head_size; - const BQ = cfg.bq; - const TILE_KV = cfg.tile_kv; - const NUM_THREADS = cfg.num_threads; - const int_arg: ArgSpec = .{ .constant = .int }; - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; - const a = try b.declareArgs(.{ - .out = .{ .buffer = .{ .type = T } }, - .q = .{ .buffer = .{ .type = T, .const_ = true } }, - .k = .{ .buffer = .{ .type = T, .const_ = true } }, - .v = .{ .buffer = .{ .type = T, .const_ = true } }, - .token_index = int_in, - .live = int_in, - .num_kv_heads = int_arg, - .scale = .{ .constant = .float }, - .q_len = int_arg, - .k_len = int_arg, - .shared_mem = .{ .threadgroup = .char }, - .tgp = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tpt = .{ .thread_position_in_threadgroup = .uint3 }, - .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, - .lane = .{ .thread_index_in_simdgroup = .uint }, - }); - - const NUM_SIMD_LANES = 32; - const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); - const TD = @divExact(HEAD_SIZE, 8); - const TK = @divExact(TILE_KV, 8); - if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); - const unroll: metal.LoopOpts = .{ .unroll = .enable }; - - const thread_idx = a.tpt.x().to(.int); - const head_idx = a.tgp.x().to(.int); - const q_pos_start = a.tgp.y().to(.int).mul(BQ); - const batch = a.tgp.z().to(.int); - const num_heads = a.tgpg.x().to(.int); - const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); - const q_stride = num_heads.mul(HEAD_SIZE); - const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); - - const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); - const live = a.live.load(0); - const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); - const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - - var dead = b.openIf(valid_q.le(0)); - { - var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = zero.iv; - out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); - zero.close(); - } - b.return_(); - dead.close(); - } - - // Query i of the batch row is at position context_len + i. - const context_len = a.token_index.load(batch); - const seq_len = context_len.add(live); - const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); - - const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); - const LD = HEAD_SIZE + SMEM_PAD; - const Q_ELEMS = BQ * LD; - const KV_ELEMS = TILE_KV * LD; - - const Q_smem = a.shared_mem.to(T); - const K_smem = Q_smem.add(Q_ELEMS); - const V_smem = K_smem.add(KV_ELEMS); - - const log2e: f32 = std.math.log2e; - const scale_log2 = a.scale.mul(log2e); - - const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); - { - const i = load_q.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - var value = b.openIfElse(r.lt(valid_q), .{T}); - { - value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); - } - { - value.yieldElse(.{b.constant(T, 0)}); - } - Q_smem.store(r.mul(LD).add(d), value.results[0]); - load_q.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const T2: Type = .vec(T.scalar, 2); - const q_reg = b.array(T2, @intCast(TD)); - var load_q_reg = b.openForOpts(0, TD, 1, unroll); - { - const d = load_q_reg.iv; - const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - q_reg.store(d, tmp.threadElements()); - load_q_reg.close(); - } - - const fc = fragCoord(b, a.lane.to(.ushort)); - const fm = fc.y(); - const fn_ = fc.x(); - - const max_score = b.local(.float, masked_score); - const sum_score = b.local(.float, 0.0); - - const o_reg = b.array(.float2, @intCast(TD)); - var zero_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(zero_o.iv, b.constant(.float2, 0)); - zero_o.close(); - } - - const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); - const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); - - var tiles = b.openFor(0, num_kv_tiles, 1); - { - const tile_idx = tiles.iv; - const tile_start = tile_idx.mul(TILE_KV); - - var past = b.openIf(tile_start.gt(last_q_pos)); - { - tiles.breakLoop(); - past.close(); - } - - const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; - const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); - const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; - std.debug.assert(@rem(LD, VEC) == 0); - var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); - { - const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); - const kv_pos = tile_start.add(t); - var present = b.openIfElse(kv_pos.lt(seq_len), .{}); - { - const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); - const k_ptr = a.k.add(off); - const v_ptr = a.v.add(off); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); - loop.close(); - } - present.elseBranch(); - } - { - const zero = b.constant(LoadUnit, 0); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - loop.close(); - } - present.close(); - } - rows.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const s_reg = b.array(.float2, @intCast(TK)); - var zero_s = b.openForOpts(0, TK, 1, unroll); - { - s_reg.store(zero_s.iv, b.constant(.float2, 0)); - zero_s.close(); - } - - var qk_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = qk_dims.iv; - const q_frag = b.matrix(T.scalar, q_reg.load(d)); - var qk_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = qk_keys.iv; - const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); - const s_frag = b.matrix(.float, s_reg.load(k)); - s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); - qk_keys.close(); - } - qk_dims.close(); - } - - const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); - const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); - const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); - - var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); - { - var scale = b.openForOpts(0, TK, 1, unroll); - { - const k = scale.iv; - setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); - setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); - scale.close(); - } - masking.elseBranch(); - } - { - var keys = b.openForOpts(0, TK, 1, unroll); - { - const k = keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj).mul(scale_log2); - const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); - const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); - setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); - pair.close(); - } - keys.close(); - } - masking.close(); - } - - const local_max = b.local(.float, masked_score); - var max_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = max_keys.iv; - local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); - max_keys.close(); - } - const row_max = fragRowReduce(b, .max, local_max); - - const prev_max = max_score.rvalue(); - const grown_max = b.max(prev_max, row_max); - const grew = grown_max.gt(prev_max); - const was_empty = prev_max.eq(masked_score); - const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); - const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); - // A tile masking a whole row leaves the maximum masked: zero would - // pin it, and later scores far below zero would underflow. - max_score.assign(grown_max); - - const local_sum = b.local(.float, 0.0); - var p_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = p_keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj); - const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); - setLane(s_reg, k, jj, p); - local_sum.assign(local_sum.add(p)); - pair.close(); - } - p_keys.close(); - } - const row_sum = fragRowReduce(b, .sum, local_sum); - - sum_score.assign(sum_score.mul(factor).add(row_sum)); - - var rescale_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); - rescale_o.close(); - } - - var pv_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = pv_keys.iv; - const p_frag = b.matrix(.float, s_reg.load(k)); - var pv_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = pv_dims.iv; - const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - const o_frag = b.matrix(.float, o_reg.load(d)); - o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); - pv_dims.close(); - } - pv_keys.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - tiles.close(); - } - - const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); - var normalize = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); - normalize.close(); - } - - const O_smem = Q_smem.to(.float); - var store_o = b.openForOpts(0, TD, 1, unroll); - { - const d = store_o.iv; - const o_frag = b.matrix(.float, o_reg.load(d)); - b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - store_o.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = write_out.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); - out_base.store(r.mul(q_stride).add(d), value); - write_out.close(); - } - } -}; - -test "the flash attention kernels emit verified modules" { - const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); - } - } - for ([_]Type{ .half, .bfloat }) |dtype| { - for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { - const tile = FaTiled.tileConfig(head_size).?; - allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); - } - } -} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index bdfb00c0..9327c2a1 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,10 +1,12 @@ const std = @import("std"); +const stdx = @import("stdx"); + const zml = @import("../../../zml.zig"); -const flash_attn = @import("flash_attn.zig"); + +const log = std.log.scoped(.@"zml/attention/metal"); pub const Metadata = struct { - /// Live queries of a single-sequence prefill; 0 means all. num_tokens: zml.Tensor, pub fn init() Metadata { @@ -21,123 +23,16 @@ pub const Metadata = struct { } }; -/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for -/// unsupported dtypes and head sizes. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const dtype = q.dtype(); - const head_dim = q.dim(.hd); - if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { - const seq_len = k.dim(.k); - const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) - .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); - return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); - } + const qc = q.transpose(.{ .h, .q, .hd }); + const kc = k.transpose(.{ .h, .k, .hd }); + const vc = v.transpose(.{ .h, .k, .hd }); + const tok_i32 = token_index.convert(.i32); - const batched = q.shape().hasTag(.b) != null; - const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); - const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); - const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); - - const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; - const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) - decode(qb, kb, vb, positions, scale) + const attn = if (q.dim(.q) > 1) + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) else - prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); - return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); -} + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); -fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { - const all: zml.Tensor = .scalar(q.dim(.q), .i32); - if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; - const num_tokens = metadata.num_tokens.convert(.i32); - return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); -} - -/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { - const batch: i32 = @intCast(q.dim(.b)); - const num_heads: i32 = @intCast(q.dim(.h)); - const head_dim: i32 = @intCast(q.dim(.hd)); - const num_kv_heads: i32 = @intCast(k.dim(.h)); - const keys: i32 = @intCast(k.dim(.k)); - const nrows = batch * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); - const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; - - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ - .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; - - return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, - .constants = .{ .args = .of(args) }, - }).dst; -} - -/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { - const queries: i32 = @intCast(q.dim(.q)); - const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; - const cfg: flash_attn.FaTiled.Cfg = .{ - .dtype = zml.kernel.metal.from(q.dtype()), - .head_size = @intCast(q.dim(.hd)), - .bq = tile[0], - .tile_kv = tile[1], - .num_threads = tile[2], - }; - return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, - .threadgroup = .{ @intCast(tile[2]), 1, 1 }, - .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), - .constants = .{ - .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), - .scale = .of(scale), - .q_len = .of(queries), - .k_len = .of(@as(i32, @intCast(k.dim(.k)))), - }, - }).out; -} - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} - -test { - std.testing.refAllDecls(flash_attn); + return attn.transpose(q.shape()); } diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index f98a545d..282da132 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,17 +43,71 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const decode = q.dim(.q) == 1; - const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; - const options: zml.ops.NeuronNkiOps = .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = parameters.compiler_target, - }; - const output = if (decode) - zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] - else - zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; - return output.convert(q.dtype()); + // TODO: https://github.com/zml/zml/issues/914 + const tp = zml.Compiler.current().resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(tp, .{ .h = .model }); + const k_sharded = k.withPartitioning(tp, .{ .h = .model }); + const v_sharded = v.withPartitioning(tp, .{ .h = .model }); + + if (q.dim(.q) == 1) { + const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) + .withPartitioning(tp, .{ .row = .replicated, .col = .replicated }); + + return zml.ops.manualComputation( + (struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + parameters: Parameters, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + const kernel = self.parameters.decode_kernel; + return zml.ops.neuronNki( + .{ self.q, self.k, self.v, self.token_index }, + .{output}, + .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = self.parameters.compiler_target, + }, + )[0]; + } + }).body, + .{ + .q = q_sharded, + .k = k_sharded, + .v = v_sharded, + .token_index = token_index_2d, + .parameters = parameters, + }, + q_sharded.shape(), + ).convert(q.dtype()); + } + + return zml.ops.manualComputation( + (struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + const kernel = self.parameters.prefill_kernel; + return zml.ops.neuronNki( + .{ self.q, self.k, self.v }, + .{output}, + .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = self.parameters.compiler_target, + }, + )[0]; + } + }).body, + .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, + q_sharded.shape(), + ).convert(q.dtype()); } diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig deleted file mode 100644 index c3c3dca2..00000000 --- a/zml/kernels/flash_attention/tests.zig +++ /dev/null @@ -1,346 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const flash_attention = @import("flash_attention.zig"); - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { - try testAttention( - .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), - .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), - &.{3000}, - ); -} - -test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), - .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), - &.{900}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); - const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer flash_attention.Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig deleted file mode 100644 index b66593c1..00000000 --- a/zml/kernels/moe/api.zig +++ /dev/null @@ -1,164 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const Tensor = @import("../../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/moe"); - -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, - - pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { - const x = input.convert(.f32); - return switch (self) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - .row_interleaved_8 => blk: { - const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); - break :blk .{ - grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - }; - }, - }; - - break :b switch (self) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; - } -}; - -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -pub const ProjectionLayout = enum { - concatenated, // G0 G1 … Gn U0 U1 … Un - row_interleaved_1, // G0 U0 G1 U1 … - row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … -}; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Gate/up ordering before applying the backend-specific packing. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte - /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, - /// represented as `[expert, k/128, rows]` i32 words of four scales. - mxfp4_interleaved8, - }; -}; - -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: Tensor, - weights: Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?Tensor { - const start = self.expert_start orelse return null; - const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 8bc03c9c..c0156720 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,9 +3,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../api.zig").Options; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -27,7 +24,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: Options) void { +pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -38,14 +35,14 @@ pub fn validateOptions(opts: Options) void { /// is the per-expert multiplier dequantizing each GEMM output (it folds in /// the inverse of the input's global scale), and their input scale is the /// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows -/// interleave by 8 and its scale atoms are MN-major packed i32 words. +/// expect block scales in the 128x4 tensor-core layout. pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + ids: zml.Tensor, + weights: zml.Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: Options, + options: zml.moe.Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); @@ -86,43 +83,24 @@ pub fn fusedExperts( else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), }; - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); - const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - - return switch (format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .ids = ids, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - }), - }.reshape(input.shape().dims()).withTags(input.shape()); + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .topk = ids.dim(.topk), + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + .format = format, + .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), + }; + return if (context.expert_parallel) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); } /// The format-specific tensors: NVFP4's per-expert global scales. They stay @@ -137,3 +115,69 @@ const Format = union(enum) { act2: zml.Tensor, }, }; + +const Context = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + topk: i64, + swiglu_limit: ?f32, + routing_weight_placement: zml.moe.RoutingWeightPlacement, + format: Format, + expert_parallel: bool, + + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + const tokens = self.input.dim(.t); + var ids = self.ids.convert(.i32); + const routing_weights = self.weights.convert(.f32); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(experts); + const expert_end = expert_start.addConstant(experts); + // Routes of other ranks get expert -1 and are not computed here. + const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); + ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); + } + const result = switch (self.format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ + .x = self.input, + .routing_weights = routing_weights, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ + .x = self.input, + .routing_weights = routing_weights, + .ids = ids, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = self.swiglu_limit, + .routing_weight_placement = self.routing_weight_placement, + }), + }.withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; + } +}; diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig index da639c75..47e05d37 100644 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -138,7 +138,7 @@ pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) Pipeli pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { var peek = b.openIfElse(count.lt(limit), .{.i1}); { - peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); + peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); } { peek.yieldElse(.{b.constant(.i1, true)}); diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index ff6edec2..479e6516 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,31 +4,26 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. x: zml.Tensor, /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, - /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. w1: zml.Tensor, - /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, - /// in MN-major 128x4 atoms, with rows interleaved like `w1`. + /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s1: zml.Tensor, - /// `[experts, hidden, intermediate / 2]` packed FP4. w2: zml.Tensor, - /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. + /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, @@ -212,14 +207,14 @@ pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, t .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, - .weight_scale = a.s1, + .weight_scale = a.s1.bitCast(.u8), .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, - .weight_scale = a.s2, + .weight_scale = a.s2.bitCast(.u8), .input_scale = q2.s, }, routing.route_inverse); return reduce.topk(down, routing.route_map orelse routing.schedule, .{ @@ -257,7 +252,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: RoutingWeightPlacement = .after_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index 30062a3a..e8901fb1 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -550,7 +550,7 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .weight = .{ .ptr = cute.DType.i8 }, .schedule = .{ .ptr = cute.DType.i32 }, .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i32 }, + .weight_scale = .{ .ptr = cute.DType.i8 }, .input_scale = .{ .ptr = cute.DType.i8 }, }; const shape = .{ @@ -762,10 +762,10 @@ fn issueTmaStage( b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); } - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -826,7 +826,10 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.mma_barriers, first_state.index), + first_state.phase, + ); var stages = b.openFor( 0, k_tiles, @@ -842,7 +845,7 @@ fn runTmaProducer( const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); pending.close(); } issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); @@ -890,7 +893,11 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity( + barrierAt(b, storage.mma_barriers, tail_state.index), + tail_state.phase, + 10_000_000, + ); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -915,8 +922,8 @@ const Mxfp4Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); } }; @@ -1157,14 +1164,17 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.load_barriers, first_state.index), + first_state.phase, + ); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1182,7 +1192,7 @@ fn runMmaConsumer( const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); pending.close(); } const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); @@ -1199,7 +1209,7 @@ fn runMmaConsumer( ); var commit_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); commit_lane.close(); } const next_count = stages.carried[2].add(1); @@ -1210,7 +1220,7 @@ fn runMmaConsumer( var full_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); full_lane.close(); } const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); @@ -1261,7 +1271,11 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity( + barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), + accumulator_tail.phase, + 10_000_000, + ); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1335,7 +1349,7 @@ fn runEpilogue( .phase = tiles.after_carried[5], }; const stage = accumulator_state.index; - b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1361,7 +1375,7 @@ fn runEpilogue( if (chunk + 1 == tile_chunks) { // Registers now own the accumulator values, so all 128 epilogue // threads release this TMEM stage before the stores. - _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); @@ -1391,7 +1405,7 @@ fn runEpilogue( b.relinquishTmemAllocPermit(.{}); release_warp.close(); } - b.barrierCtaSyncAligned(1, 128); + b.namedBarrier(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); { b.deallocTmem(tmem, tile.tmem_columns, .{}); @@ -1439,10 +1453,11 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, return .{ .count = count, .route = route, .weight = weight }; } -/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 -/// projection. Gate and up lanes are paired by XOR 8. Each gate thread -/// computes one SwiGLU activation per routed row; 32 consecutive activations -/// span two warps, whose partial maxima meet in shared memory. +/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` +/// of the interleaved projection, so even threads hold a gate and their odd +/// neighbour the matching up value. Each even thread computes one SwiGLU +/// activation per routed row; 32 consecutive activations span two warps, +/// whose partial maxima meet in shared memory. /// /// At decode sizes most rows of a group are padding, whose outputs are never /// read. `count` is uniform across the CTA, so whole 8-row chunks of padding @@ -1469,7 +1484,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const tid = b.threadIdx().x; const lane = tid.rem(32); const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(16).lt(8); + const is_gate = tid.rem(2).eq(0); const first_row = group.mul(operands.group_rows); const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. @@ -1496,7 +1511,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo if (operands.swiglu_limit) |limit| { gate = gate.minimum(limit); } - var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { up_value = up_value.maximum(-limit).minimum(limit); } @@ -1505,7 +1520,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Up lanes hold no activation and must not raise the maxima. + // Odd threads hold no activation and must not raise the maxima. activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); @@ -1530,17 +1545,16 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo scratch.set(.{ warp, lane }, lane_maximum); owner.close(); } - b.barrierCtaSyncAligned(1, 128); + b.namedBarrier(1, 128); // Lane `c` turns the two half-block maxima of column `c` into its scale. const pair = warp.bitAnd(~@as(i32, 1)); const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); // The next tile reuses the scratch maxima. - b.barrierCtaSyncAligned(1, 128); + b.namedBarrier(1, 128); - const activation_row = tid.div(16).mul(8).add(tid.rem(8)); - const activation_index = tile_m.mul(64).add(activation_row); + const activation_index = tile_m.mul(64).add(tid.div(2)); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1753,7 +1767,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var mma_warp = b.openIf(warp.eq(4)); { // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.barrierCtaSyncAligned(2, 160); + b.namedBarrier(2, 160); switch (k_tiles) { inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), else => unreachable, @@ -1770,7 +1784,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); } - b.barrierCtaSyncAligned(2, 160); + b.namedBarrier(2, 160); switch (epilogue) { .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ .group_sizes = view(b, d.group_sizes), @@ -1830,13 +1844,7 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - // Weight scales step M atoms first; activation scales retain K-major order. - const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); - const sfa_layout = b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, - ); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig index 0de45007..ac59735c 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -699,10 +699,10 @@ fn issueTmaStage( var elected = b.openIf(b.electSync()); b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -756,7 +756,10 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.mma_barriers, first_state.index), + first_state.phase, + ); var stages = b.openFor( 0, @as(i32, @intCast(k_tiles)), @@ -770,7 +773,7 @@ fn runTmaProducer( }; const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); pending.close(); issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); const next_count = stages.carried[1].add(1); @@ -814,7 +817,11 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity( + barrierAt(b, storage.mma_barriers, tail_state.index), + tail_state.phase, + 10_000_000, + ); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -836,8 +843,8 @@ const Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); } }; @@ -1013,14 +1020,17 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.load_barriers, first_state.index), + first_state.phase, + ); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1033,7 +1043,7 @@ fn runMmaConsumer( const k = stages.carried[2]; const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); pending.close(); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); @@ -1048,7 +1058,7 @@ fn runMmaConsumer( mainloop.accumulator, ); var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); commit_lane.close(); const next_count = stages.carried[2].add(1); const next_state = advancePipeline(b, state.index, state.phase, tile.stages); @@ -1056,7 +1066,7 @@ fn runMmaConsumer( stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); full_lane.close(); const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); const next_acc_count = tiles.after_carried[8].add(1); @@ -1103,7 +1113,11 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity( + barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), + accumulator_tail.phase, + 10_000_000, + ); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1179,7 +1193,7 @@ fn runEpilogue( const expert = operands.expert_ids.get(.{group}, .i32); const alpha = operands.alpha.get(.{expert}, .f32); const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; - b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1199,7 +1213,7 @@ fn runEpilogue( const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); b.fenceTmemLoad(); if (chunk + 1 == tile_chunks) { - _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); @@ -1225,7 +1239,7 @@ fn runEpilogue( var release_warp = b.openIf(warp.eq(0)); b.relinquishTmemAllocPermit(.{}); release_warp.close(); - b.barrierCtaSyncAligned(1, 128); + b.namedBarrier(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); b.deallocTmem(tmem, tile.tmem_columns, .{}); dealloc_warp.close(); @@ -1529,7 +1543,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E tma_warp.close(); var mma_warp = b.openIf(warp.eq(4)); - b.barrierCtaSyncAligned(2, 160); + b.namedBarrier(2, 160); runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); mma_warp.close(); @@ -1537,7 +1551,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var alloc_warp = b.openIf(warp.eq(0)); b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); - b.barrierCtaSyncAligned(2, 160); + b.namedBarrier(2, 160); const shared_operands: EpilogueOperands = .{ .expert_ids = expert_ids, .alpha = view(b, d.alpha), diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel new file mode 100644 index 00000000..54b8604c --- /dev/null +++ b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cutlass_flashinfer", + srcs = [ + "cutlass_flashinfer.zig", + ], +) diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig new file mode 100644 index 00000000..2c7ad342 --- /dev/null +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -0,0 +1,633 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const zml = @import("../../../zml.zig"); + +const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32, + activation: zml.moe.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32, + + pub fn fromGlobalOptions(opts: zml.moe.Options) Options { + return .{ + .workspace_query_device = 0, + .activation = opts.activation, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, + }; + } +}; + +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + +const Input = struct { + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, +}; + +const Output = struct { + output: zml.Shape, + workspace: zml.Shape, +}; + +const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +fn runnersFromAttributes(attributes: Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +fn currentRunners() !*Runners { + const platform = zml.Compiler.current().platform; + const runners = platform.state.cuda.fi_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: zml.moe.Activation) i32 { + return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + }; +} + +fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +fn ffiCallNvfp4( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.pjrtx.TensorToCustomCallBuffer(Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), + attributes: Attributes, +) !?*zml.pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = input.hidden_states.ptr; + io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); + io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); + io.fc1_expert_weights = input.fc1_weights.ptr; + io.fc2_expert_weights = input.fc2_weights.ptr; + io.quant_scales[0] = input.fc1_act_global.ptr; + io.quant_scales[1] = input.fc1_weight_block.ptr; + io.quant_scales[2] = input.fc1_global.ptr; + io.quant_scales[3] = input.fc2_act_global.ptr; + io.quant_scales[4] = input.fc2_weight_block.ptr; + io.quant_scales[5] = input.fc2_global.ptr; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = output.output.ptr; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = output.workspace.ptr; + workspace.data_bytes = output.workspace.shape.byteSize(); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ + .name = "flashinfer_cutlass_nvfp4_routed_moe", + // Expert mesh is owned by forwardMoe outer manual computation. + .sharding_aware = false, + .has_side_effect = false, +}); + +fn computeCapability(platform: *const zml.Platform) !u16 { + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; +} + +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, +) !void { + if (comptime platforms.isEnabled(.cuda)) { + var cuda_state = &platform.state.cuda; + if (cuda_state.fi_cutlass_moe_runners != null) return; + try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); + const runners = try allocator.create(Runners); + runners.* = .{}; + cuda_state.fi_cutlass_moe_runners = runners; + return; + } + return error.UnsupportedPlatform; +} + +pub fn register(platform: *const zml.Platform) !void { + if (comptime platforms.isEnabled(.cuda)) { + try routedNvfp4Call.register(platform); + return; + } + return error.UnsupportedPlatform; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; +} + +pub fn tacticCounts( + platform: *const zml.Platform, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: zml.moe.Activation, +) zml.Shape { + const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 2 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 2 or + topk_ids.rank() != 2) + { + return error.InvalidShape; + } + + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + }; + const top_k = topk_ids.dim(.topk); + + if (tokens <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or + !fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = tokens, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, + options, + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + + attributes.runners = @intFromPtr(runners); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const context = makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); + + const result = routedNvfp4Call.call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const runner_options: Options = .fromGlobalOptions(opts); + const expert_partition = gate_up.weight.shape().partition(.expert); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); + + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + if (expert_partition.isSharded()) { + return zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up_weight_unpacked: zml.Tensor, + down_weight_unpacked: zml.Tensor, + gate_up_input_scale: zml.Tensor, + gate_up_scales: zml.Tensor, + gate_up_global_scale: zml.Tensor, + down_input_scale: zml.Tensor, + down_scales: zml.Tensor, + down_global_scale: zml.Tensor, + activation: zml.moe.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + + fn body( + self: @This(), + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = self.topk_ids + .cmp(.GE, expert_start) + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + self.topk_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_topk_weights = local_route_mask.select( + self.topk_weights, + zml.Tensor.scalar(-1, self.topk_weights.dtype()), + ); + + const local_output = fusedExpertsNvfp4( + self.input, + self.gate_up_weight_unpacked, + self.down_weight_unpacked, + local_topk_weights, + local_topk_ids, + self.gate_up_input_scale, + self.gate_up_scales, + self.gate_up_global_scale, + self.down_input_scale, + self.down_scales, + self.down_global_scale, + .{ + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, + }, + ); + const local_reshaped = local_output + .reshape(self.input.shape().dims()) + .withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .gate_up_weight_unpacked = gate_up_weight_unpacked, + .down_weight_unpacked = down_weight_unpacked, + .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), + .gate_up_scales = gate_up.quantization.?.scales, + .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), + .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), + .down_scales = down.quantization.?.scales, + .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), + ); + } + + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); +} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel deleted file mode 100644 index c9c5d9a6..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "flashinfer_cutlass", - srcs = [ - "contract.zig", - "flashinfer_cutlass.zig", - "runtime.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig deleted file mode 100644 index b4b77eaa..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/contract.zig +++ /dev/null @@ -1,40 +0,0 @@ -pub fn Inputs(comptime T: type) type { - return struct { - hidden_states: T, - fc1_weights: T, - fc2_weights: T, - topk_weights: T, - topk_ids: T, - fc1_act_global: T, - fc1_weight_block: T, - fc1_global: T, - fc2_act_global: T, - fc2_weight_block: T, - fc2_global: T, - }; -} - -pub fn Outputs(comptime T: type) type { - return struct { - output: T, - workspace: T, - }; -} - -pub const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; - -pub const compute_on_host: bool = false; -pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig deleted file mode 100644 index 92317c7c..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ /dev/null @@ -1,355 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; -const contract = @import("contract.zig"); -const Attributes = contract.Attributes; -const runtime = @import("runtime.zig"); - -const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); - -pub const auto_tactic: i32 = -1; - -pub const Options = struct { - /// Device used at graph-construction time to select the architecture - /// library and size the XLA-owned scratch buffer. - workspace_query_device: i32, - activation: Activation, - enable_pdl: bool, - gemm1_tactic: i32, - /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to - /// obtain the first valid GEMM2 index. - gemm2_tactic: i32, - - pub fn fromGlobalOptions(opts: GlobalOptions) Options { - return .{ - .workspace_query_device = 0, - .activation = opts.activation, - .enable_pdl = false, - .gemm1_tactic = auto_tactic, - .gemm2_tactic = auto_tactic, - }; - } -}; - -pub fn validateOptions(opts: GlobalOptions) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); -} - -const Inputs = contract.Inputs(zml.Tensor); - -const Outputs = contract.Outputs(zml.Shape); - -fn currentRunners() !*runtime.Runners { - const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; - return runners orelse error.RunnersNotLoaded; -} - -fn cutlassActivation(activation: Activation) i32 { - return switch (activation) { - .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, - .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, - .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, - .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, - .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, - .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, - .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, - }; -} - -pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { - const opts: zml.ops.CustomCallOptions = .{ - .has_side_effect = false, - .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), - .compute_on_host = contract.compute_on_host, - }; - - return zml.ops.typedCustomCall( - contract.name, - opts, - input_tensors, - output_shapes, - attributes, - ); -} - -fn computeCapability(platform: *const zml.Platform) !u16 { - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; - return switch (cc.sm()) { - 100, 103, 120 => |sm| sm, - else => error.UnsupportedArchitecture, - }; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - _ = computeCapability(platform) catch return false; - return true; -} - -pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - const compute_capability = computeCapability(platform) catch return false; - return switch (compute_capability) { - 100, 103, 120 => true, - else => false, - }; -} - -fn roundUp(value: i64, alignment: i64) i64 { - return @divTrunc(value + alignment - 1, alignment) * alignment; -} - -pub fn fc1BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, - activation: Activation, -) zml.Shape { - const rows = switch (activation) { - .silu, .gelu, .relu => intermediate_size, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, - }; - return .init( - .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, - .f8e4m3fn, - ); -} - -pub fn fc2BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, -) zml.Shape { - return .init( - .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, - .f8e4m3fn, - ); -} - -fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { - return tensor.dtype() == .f32 and - (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); -} - -fn validateInputs( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) !Attributes { - if (hidden_states.dtype() != .bf16 or - fc1_weights.dtype() != .f4e2m1 or - fc2_weights.dtype() != .f4e2m1 or - topk_weights.dtype() != .f32 or - topk_ids.dtype() != .i32 or - fc1_weight_block.dtype() != .f8e4m3fn or - fc2_weight_block.dtype() != .f8e4m3fn or - fc1_global.dtype() != .f32 or - fc2_global.dtype() != .f32) - { - return error.UnsupportedType; - } - if (hidden_states.rank() != 2 or - fc1_weights.rank() != 3 or - fc2_weights.rank() != 3 or - topk_weights.rank() != 2 or - topk_ids.rank() != 2) - { - return error.InvalidShape; - } - - const tokens = hidden_states.dim(.t); - const hidden_size = hidden_states.dim(.d); - const num_experts = fc1_weights.dim(0); - const fc1_rows = fc1_weights.dim(1); - const intermediate_size = switch (options.activation) { - .silu, .gelu, .relu => fc1_rows, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), - }; - const top_k = topk_ids.dim(.topk); - - if (tokens <= 0 or hidden_size <= 0 or - num_experts <= 0 or intermediate_size <= 0 or - @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or - top_k <= 0 or top_k > num_experts) - { - return error.InvalidShape; - } - if (fc1_weights.dim(2) != hidden_size or - fc2_weights.dim(0) != num_experts or - fc2_weights.dim(1) != hidden_size or - fc2_weights.dim(2) != intermediate_size or - topk_weights.dim(.t) != tokens or - topk_weights.dim(.topk) != top_k or - topk_ids.dim(.t) != tokens or - !fc1_weight_block.shape().eql(fc1BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - options.activation, - )) or - !fc2_weight_block.shape().eql(fc2BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - )) or - !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or - !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or - !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or - !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) - { - return error.InvalidShape; - } - - return .{ - .num_tokens = tokens, - .hidden_size = hidden_size, - .intermediate_size = intermediate_size, - .num_experts = @intCast(num_experts), - .top_k = @intCast(top_k), - .activation = cutlassActivation(options.activation), - .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = fc1_act_global.rank() == 1, - .fc2_act_per_expert = fc2_act_global.rank() == 1, - .gemm1_tactic = options.gemm1_tactic, - .gemm2_tactic = options.gemm2_tactic, - }; -} - -/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. -/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout -/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. -pub fn fusedExpertsNvfp4( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) zml.Tensor { - const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); - var attributes = validateInputs( - hidden_states, - fc1_weights, - fc2_weights, - topk_weights, - topk_ids, - fc1_act_global, - fc1_weight_block, - fc1_global, - fc2_act_global, - fc2_weight_block, - fc2_global, - options, - ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); - - attributes.runners = @intFromPtr(runners); - const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = runtime.makeContext(attributes); - var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); - requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - runtime.checkStatus( - deviceRunner.api, - deviceRunner.api.getWorkspaceRequirements( - deviceRunner.runner, - &context, - &requirements, - ), - ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - - const result = call( - .{ - .hidden_states = hidden_states, - .fc1_weights = fc1_weights, - .fc2_weights = fc2_weights, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .fc1_act_global = fc1_act_global, - .fc1_weight_block = fc1_weight_block, - .fc1_global = fc1_global, - .fc2_act_global = fc2_act_global, - .fc2_weight_block = fc2_weight_block, - .fc2_global = fc2_global, - }, - .{ - .output = hidden_states.shape(), - .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), - }, - attributes, - ); - return result.output; -} - -pub fn fusedExperts( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: GlobalOptions, -) zml.Tensor { - if (comptime !platforms.isEnabled(.cuda)) { - @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); - } - - if (gate_up.bias != null or down.bias != null) { - @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); - } - - const topk_ids = routing.localIds(); - // The runner ignores routes whose local expert id is -1. - const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - - const runner_options: Options = .fromGlobalOptions(opts); - - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - - const gate_up_weight_unpacked = unpackedWeight(gate_up); - const down_weight_unpacked = unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - return fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); -} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig deleted file mode 100644 index 028ef892..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/runtime.zig +++ /dev/null @@ -1,223 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const pjrt = @import("pjrt"); - -const DataType = @import("../../../dtype.zig").DataType; -const contract = @import("contract.zig"); - -const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); - -// TODO(Corentin): This was coming from Platform but we are now downstream. -// Find a way to solve that. -// Maybe the max_num_devices could be its own module because it only depends on Target. -const max_num_devices = 64; - -// TODO(Corentin): replace that with Shape when its dependencies are untangled and -// it doesn't depend on the platform anymore. -fn count(buffer: *const pjrt.ffi.Buffer) usize { - if (buffer.rank == 0) return @intCast(buffer.dims()[0]); - - var c: usize = 1; - for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); - - return c; -} - -fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { - const dt: DataType = switch (buffer.dtype) { - .invalid => @panic("Found an invalid pjrt buffer"), - .token => @panic("Found an token pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; - return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; -} - -pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - // TODO(Corentin): Find a place to abstract that - var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.args.buffers()[i]; - @field(input_buffers, field_name) = buf; - } - - var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.results.buffers()[i]; - @field(output_buffers, field_name) = buf; - } - - const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { - .void => {}, - .@"struct" => blk: { - var attrs: contract.Attributes = undefined; - inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { - // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); - } - break :blk attrs; - }, - else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), - }; - - return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| - pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); -} - -fn handlerInner( - call_frame: *pjrt.ffi.CallFrame, - inputs: contract.Inputs(*const pjrt.ffi.Buffer), - outputs: contract.Outputs(*const pjrt.ffi.Buffer), - attributes: contract.Attributes, -) !?*pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = inputs.hidden_states.data; - io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); - io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); - io.fc1_expert_weights = inputs.fc1_weights.data; - io.fc2_expert_weights = inputs.fc2_weights.data; - io.quant_scales[0] = inputs.fc1_act_global.data; - io.quant_scales[1] = inputs.fc1_weight_block.data; - io.quant_scales[2] = inputs.fc1_global.data; - io.quant_scales[3] = inputs.fc2_act_global.data; - io.quant_scales[4] = inputs.fc2_weight_block.data; - io.quant_scales[5] = inputs.fc2_global.data; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = outputs.output.data; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = outputs.workspace.data; - workspace.data_bytes = byteSize(outputs.workspace); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), - ); - return null; -} - -pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} - -pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -pub fn tacticCounts( - runners: *const Runners, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { - try fi_cutlass_moe.load(allocator, io, sm); -} diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index ba3d3c00..e5c5b6de 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,13 +6,9 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -21,19 +17,90 @@ test { } pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); - const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -50,10 +117,10 @@ pub fn fusedExperts( }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -81,13 +148,13 @@ pub fn fusedExperts( stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| + if (opts.expert_map) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (expert_map_) |expert_map| + const expert_ids = if (opts.expert_map) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -97,9 +164,9 @@ pub fn fusedExperts( const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = gate_up_.bias, + .bias = opts.gate_up.bias, .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), + .weight_scale = opts.gate_up.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -108,7 +175,7 @@ pub fn fusedExperts( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = opts.activation.applyActivation(gate_up_out, layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -118,9 +185,9 @@ pub fn fusedExperts( const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = down_.bias, + .bias = opts.down.bias, .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), + .weight_scale = opts.down.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -136,18 +203,18 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { - if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or - down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or +pub fn supports(opts: FusedExpertsArgs) bool { + if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or + opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or opts.routing_weight_placement != .before_down) return false; - const gate_up = gate_up_.weight; - const down = down_.weight; + const gate_up = opts.gate_up.weight; + const down = opts.down.weight; if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = gate_up_.quantizationScales() orelse return false; - const down_scale = down_.quantizationScales() orelse return false; - const experts = routing.global_num_experts; + const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; + const down_scale = opts.down.quantizationScales() orelse return false; + const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); if (experts < 1 or experts > 512) return false; - var max_width: i64 = hidden_states.dim(.d); + var max_width: i64 = opts.hidden_states.dim(.d); inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { @@ -166,7 +233,7 @@ pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.L else => return false, } } - const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); + const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index f835c889..e129c76c 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,17 +1,21 @@ const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -fn validateOptions(opts: GlobalOptions) void { +pub const Options = struct { + activation: zml.moe.Activation, + global_num_experts: i64 = -1, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_global_scale: ?Tensor = null, + w2_global_scale: ?Tensor = null, +}; + +fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -68,30 +72,57 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso } pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); + stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); - const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up = unpackedWeight(gate_up_); - const down = unpackedWeight(down_); + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + return fusedExpertsImpl( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + .{ + .activation = opts.activation, + .global_num_experts = gate_up_weight_unpacked.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_global_scale = gate_up_global_scale, + .w2_global_scale = down_global_scale, + }, + ); +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + gate_up: Tensor, + down: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + opts: Options, +) Tensor { stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { .bf16, .f16, .f32 => true, else => false, @@ -99,28 +130,28 @@ pub fn fusedExperts( stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { + const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { .bf16, .f16, .f32 => true, else => false, }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); - stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); + stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); const d = hidden_states.dim(.d); const num_tokens = hidden_states.dim(.t); - const topk = routing.ids.dim(.topk); + const topk = topk_ids.dim(.topk); const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { @@ -132,29 +163,29 @@ pub fn fusedExperts( const x_rows = hidden.insertAxes(.d, .{.topk}) .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }).convert(.i32); const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, - gate_up_scales, + opts.w1_scale, expert_ids, zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, - ), gate_up_global_scale, expert_ids); - const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); + ), opts.w1_global_scale, expert_ids); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, down, - down_scales, + opts.w2_scale, expert_ids, zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, - ), down_global_scale, expert_ids); + ), opts.w2_global_scale, expert_ids); - const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index eb697f36..927f95ca 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,21 +4,14 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const Activation = @import("api.zig").Activation; -pub const ActivationKind = @import("api.zig").ActivationKind; +pub const tcl = @import("tcl/tcl.zig"); +pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("api.zig").ExpertsLayout; -pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); -pub const fly = @import("fly/fly.zig"); -pub const LocalRouting = @import("api.zig").LocalRouting; +pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("api.zig").Options; -pub const ProjectionLayout = @import("api.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; -pub const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); +pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -27,6 +20,60 @@ test { _ = @import("tests.zig"); } +pub const ProjectionLayout = enum { concatenated, interleaved }; +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +/// How a backend expects the expert weights to be stored +/// The experts are loaded in this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Whether the two projections occupy contiguous halves or alternate. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, +}; + pub const Backend = enum { stablehlo, cute, @@ -38,6 +85,8 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + // Keep the dtype as an argument because non scheme-specific backends may depend on it later + _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -48,7 +97,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (flashinfer_cutlass.isNvfp4Supported(platform)) + else if (cutlass_flashinfer.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -71,13 +120,7 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => b: { - const s = scheme orelse break :b .tcl; - break :b switch (s) { - .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, - .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, - }; - }, + .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -93,7 +136,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), + .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -112,17 +155,13 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme) |s| switch (s) { - .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, - .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, - else => error.UnsupportedQuantization, - } else error.UnsupportedQuantization, + .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } @@ -130,7 +169,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => flashinfer_cutlass.register(platform), + .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -139,6 +178,14 @@ pub const Backend = enum { } }; +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -149,35 +196,6 @@ pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Te weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); } -pub const Partitioning = struct { - const Logical = zml.Sharding.Partitioning.Logical; - - in_specs: struct { - input: Logical, - ids: Logical, - weights: Logical, - gate_up: Logical, - down: Logical, - }, - out_specs: Logical, - - /// Shards the `.expert` dimension of both projections over `mesh_axis`. - /// Other dimensions, input tokens, routing IDs/weights and output are replicated. - pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { - const experts: Logical = .parse(.{ .expert = mesh_axis }); - return .{ - .in_specs = .{ - .input = .replicated, - .ids = .replicated, - .weights = .replicated, - .gate_up = experts, - .down = experts, - }, - .out_specs = .replicated, - }; - } -}; - /// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. /// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( @@ -187,112 +205,72 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, backend: Backend, - mesh: *const zml.Mesh, - partitioning: Partitioning, opts: Options, ) zml.Tensor { validateRoutingInputs(input, topk_ids, topk_weights); + return switch (backend) { + .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + }; +} - const shapes = .{ - .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), - .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), - .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), - .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), - .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), +pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} + +pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { + const x = input.convert(.f32); + return switch (activation) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (activation) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, }; - const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); - - const tokens = shapes.input.partition(.t); - const experts = shapes.gate_up.partition(.expert); - // Local kernels require complete feature vectors and top-k routing lists. - stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); - // The expert-axis reduction must add contributions for the same token rows. - // Sharding tokens along that axis would mix contributions for different tokens. - stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); - stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); - for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { - stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); - for (0..shape.rank()) |axis| { - stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); - } - } - const expert_axis = experts.meshAxis(); - const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); - - const Call = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - backend: Backend, - opts: Options, - mesh: *const zml.Mesh, - expert_axis: ?u3, - global_experts: u32, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - const routing: LocalRouting = .{ - .ids = args.ids, - .weights = args.weights, - .global_num_experts = args.global_experts, - .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), - .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, - }; - const output = switch (args.backend) { - .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - }; - return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; - } - }; - const args: Call = .{ - .input = input, - .ids = topk_ids, - .weights = topk_weights, - .gate_up = gate_up, - .down = down, - .backend = backend, - .opts = opts, - .mesh = mesh, - .expert_axis = if (parallel) expert_axis else null, - .global_experts = @intCast(gate_up.weight.dim(.expert)), - }; - const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; - for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { - zml.meta.mapAlloc(struct { - fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { - const axis = ctx.axis orelse return .replicated; - if (tensor.count() == 1) return .replicated; - // Packed scales may be untagged; experts are their leading dimension. - const dim = tensor.shape().hasTag(.expert) orelse 0; - stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); - return .{ .mapping = .{ - .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), - .tgt = .init(&.{axis}), - } }; - } - }.spec, zml.Compiler.current().arena.allocator(), .{ - .experts = linear.weight.dim(.expert), - .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, - }, linear, specs) catch @panic("OOM"); - } - return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ - .in_specs = .{ - .input = partitioning.in_specs.input, - .ids = partitioning.in_specs.ids, - .weights = partitioning.in_specs.weights, - .gate_up = projection_specs[0], - .down = projection_specs[1], - }, - .out_specs = partitioning.out_specs, - }); } diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index bb60594b..09dc360a 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,9 +5,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -15,9 +12,22 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -fn validateOptions(opts: GlobalOptions) void { +pub const Options = struct { + activation: zml.moe.Activation, + global_num_experts: i64 = -1, + expert_map: ?Tensor = null, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_bias: ?Tensor = null, + w2_bias: ?Tensor = null, +}; + +fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + // TODO(Corentin): find where to put that + //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; + //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -214,25 +224,120 @@ pub fn callGmmEp( } pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - const expert_map_ = routing.expertMap(); + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (expert_partition.isSharded()) { + const global_num_experts = down.weight.dim(.expert); + const partial_output = zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: zml.moe.Activation, + global_num_experts: i64, + gate_up_scales: ?zml.Tensor, + bias_gate_up: ?zml.Tensor, + down_scales: ?zml.Tensor, + bias_down: ?zml.Tensor, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_output = fusedExpertsImpl( + self.input, + self.weights_gate_up, + self.weights_down, + self.topk_weights, + self.topk_ids, + .{ + .activation = self.activation, + .global_num_experts = self.global_num_experts, + .expert_map = expert_map, + .w1_scale = self.gate_up_scales, + .w2_scale = self.down_scales, + .w1_bias = self.bias_gate_up, + .w2_bias = self.bias_down, + }, + ); + return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = opts.activation, + .global_num_experts = global_num_experts, + .gate_up_scales = gate_up_scales, + .bias_gate_up = gate_up.bias, + .down_scales = down_scales, + .bias_down = down.bias, + }, + input.shape(), + ); + return zml.ops.allReduce(partial_output, zml.Tensor.add); + } + + return fusedExpertsImpl( + input, + gate_up.weight, + down.weight, + topk_weights, + topk_ids, + .{ + .activation = opts.activation, + .global_num_experts = gate_up.weight.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, + }, + ); +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + w1: Tensor, + w2: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + opts: Options, +) Tensor { const tokens = hidden_states.dim(.t); const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); - const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); - const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); - const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); + const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); + const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); + const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); + const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); - const num_experts = routing.global_num_experts; - if (expert_map_) |expert_map| { + const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); + if (opts.expert_map) |expert_map| { if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } @@ -240,7 +345,7 @@ pub fn fusedExperts( const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { + const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); @@ -251,7 +356,7 @@ pub fn fusedExperts( }; const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (expert_map_ != null) + const hidden_sorted = if (opts.expert_map != null) permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) else blk: { // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route @@ -274,19 +379,19 @@ pub fn fusedExperts( .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (gate_up_.bias) |bias| { + if (opts.w1_bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); gate_up_out = gate_up_out.add(bias_per_token); } - const activated = opts.activation.applyActivation(gate_up_out, .concatenated); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (down_.bias) |bias| { + if (opts.w2_bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); down_out = down_out.add(bias_per_token); } diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index ba723234..e33d2a14 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,18 +1,13 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; -pub fn activate(x: Tensor, activation: Activation) Tensor { +pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -153,28 +148,23 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: Activation, + activation: zml.moe.Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: RoutingWeightPlacement = .after_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, }; -/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { - stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); const tokens = input.dim(.t); const k = topk_ids.dim(.topk); - // Nonlocal routes use expert zero for safe gathers; discard their full - // contribution after the down projection, including any projection bias. - const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); - const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); - const ids = safe_ids.reshape(.{ .route = tokens * k }); + stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); + const ids = topk_ids.reshape(.{ .route = tokens * k }); const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) @@ -186,15 +176,13 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); - output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { - stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); - return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ +pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { + return reference(input, ids, weights, gate_up, down, .{ .activation = opts.activation, .quantize_input = opts.quantize_input, .routing_weight_placement = opts.routing_weight_placement, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 44b56e08..4057eea3 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,8 +2,5 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = [ - "moe.zig", - "tcl.zig", - ], + srcs = ["tcl.zig"], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig deleted file mode 100644 index 23fb2274..00000000 --- a/zml/kernels/moe/tcl/moe.zig +++ /dev/null @@ -1,457 +0,0 @@ -//! TCL kernel schedules for Furiosa RNGD fused experts. -const std = @import("std"); - -const builder = @import("dsl/tcl/builder"); - -const zml = @import("../../../zml.zig"); -const tcl = zml.kernel.tcl; -const ActivationKind = @import("../api.zig").ActivationKind; - -pub const Format = enum { bf16, fp8_block128 }; - -/// Dispatch in `tcl.zig` uses the first matching schedule. -/// -/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected -/// expert on the whole batch to reuse its weights, with zero routing weight -/// for unrouted pairs. -/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather -/// weights for each route to avoid the overhead of grouping routes. -/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to -/// reuse weights within each block, then gather results back by route. -pub const Schedule = enum { per_route, blockwise, small_batch }; - -pub const Cfg = struct { - num_tokens: i64, - top_k: i64, - local_num_experts: i64, - hidden_size: i64, - expert_intermediate_size: i64, - weight_scale_dtype: tcl.DType = .bf16, - activation: ActivationKind = .swiglu, -}; - -const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; -const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; - -pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { - if (schedule == .small_batch and format != .fp8_block128) { - @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); - } - return tcl.Kernel(Cfg, .{ - .name = "moe_" ++ @tagName(schedule) ++ switch (format) { - .bf16 => "", - .fp8_block128 => "_fp8", - }, - .inputs = switch (format) { - .bf16 => base_inputs, - .fp8_block128 => base_inputs ++ scale_inputs, - }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, schedule); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, format); - switch (schedule) { - .per_route => try perRoute(b, a, in, cfg.activation, format), - .blockwise => try blockwise(b, a, in, cfg, format), - .small_batch => try fp8SmallBatch(b, a, in, cfg), - } - } - }.run, - }); -} - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.num_tokens), - .K = b.axis("K", cfg.top_k), - .E = b.axis("E", cfg.local_num_experts), - .D = b.axis("D", cfg.hidden_size), - .I = b.axis("I", cfg.expert_intermediate_size), - .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { - const weight_dtype: tcl.DType = switch (format) { - .bf16 => .bf16, - .fp8_block128 => .f8_e4, - }; - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, - }; - if (format == .bf16) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); - const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { - if (format == .bf16) return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { - if (format == .bf16) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // The builder can infer the TCL `Interleaving` tactic for two inputs with - // matching axes, such as `gate` and `up`. That tactic allows exactly one - // binary instruction, so force `.elementwise` for this activation chain. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (format == .fp8_block128) blk: { - if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); - - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// Use IO-bound tactics for per-route and FP8 small-batch schedules and -/// compute-bound tactics for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { - const config = comptime blk: { - const common = .{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .propagate_sparse_axis_from_op = "None", - }; - const tactics = switch (schedule) { - .blockwise => .{ - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - }, - .per_route, .small_batch => .{ - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - }, - }; - // The transaction-simulation setting is only emitted for IO-bound schedules. - const common_info = @typeInfo(@TypeOf(common)).@"struct"; - const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; - const Config = @Struct( - .auto, - null, - common_info.field_names ++ tactics_info.field_names, - common_info.field_types ++ tactics_info.field_types, - common_info.field_attrs ++ tactics_info.field_attrs, - ); - var merged: Config = undefined; - for (common_info.field_names) |field| @field(merged, field) = @field(common, field); - for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); - break :blk merged; - }; - b.compilerConfig(config); -} - -/// Gather the selected experts for every token. -fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); - if (format == .fp8_block128) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } -} - -/// Rows per expert block: the next power of two of the routes per expert. -fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -/// Group routes into expert-major blocks, then run both projections in a -/// device loop over chunks of blocks. -fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); -} - -/// Number of experts processed per device-loop iteration, based on the -/// batch's route count (`cfg.num_tokens * cfg.top_k`). -fn expertsPerChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. -/// Load each selected expert once and run it for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { - const chunk = expertsPerChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); - const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); -} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index f05727ce..4b7bcfb3 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,165 +1,270 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +//! +//! Few tokens (at most `experts / top_k`): activation stationary, each token +//! gathers the weights of its experts and the down projection contracts over +//! the experts and the intermediate axis at once. +//! +//! More tokens: blockwise. The kernel routes on the device: routes are placed +//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a +//! device loop over the used chunks of `BGi` blocks gathers each block's +//! expert weights and runs both projections; outputs are gathered back per +//! route and weighted. const std = @import("std"); +const builder = @import("dsl/tcl/builder"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const kernels = @import("moe.zig"); -const api = @import("../api.zig"); -/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` -/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is -/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with -/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { - const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); +/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// SwiGLU, routing weights applied after the down projection. +pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); + stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); - const format = expertFormat(gate_up_linear, down_linear); - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const cfg: kernels.Cfg = .{ - .num_tokens = input.dim(.t), - .top_k = topk_ids.dim(.topk), - .local_num_experts = gate_up.dim(.expert), - .hidden_size = input.dim(.d), - .expert_intermediate_size = down.dim(.dout), - .weight_scale_dtype = switch (format) { - .bf16 => .bf16, - .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), - }, - .activation = std.meta.activeTag(opts.activation), + stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); + const t = input.dim(.t); + const k = topk_ids.dim(.topk); + const cfg: Cfg = .{ + .t = t, + .k = k, + .e = gate_up.dim(.expert), + .d = input.dim(.d), + .i = down.dim(.dout), }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - const weights = routing.weights.convert(.f32); - const local_weights = switch (format) { - .bf16 => weights, - .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), - }; - return switch (format) { - .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), - .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ - .x = input, - .ids = topk_ids, - .w = local_weights, - .gate_up = gate_up, - .down = down, - .gate_up_scale = s.gate_up, - .down_scale = s.down, - }), - }; + const x = input.reshape(.{ t, cfg.d }); + const ids = topk_ids.reshape(.{ t, k }); + const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const out_shape = x.shape(); + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); } -const Scales = struct { gate_up: Tensor, down: Tensor }; +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; -const Format = union(kernels.Format) { - bf16: void, - fp8_block128: Scales, +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.t), + .K = b.axis("K", cfg.k), + .E = b.axis("E", cfg.e), + .D = b.axis("D", cfg.d), + .I = b.axis("I", cfg.i), + .I2 = b.axis("I2", 2 * cfg.i), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } }; -fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { - const scheme = gate_up.quantizationScheme(); - stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); - return switch (scheme orelse { - stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); - return .bf16; - }) { - .fp8_block128 => blk: { - const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - break :blk .{ .fp8_block128 = scales }; - }, - else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), - }; +fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { + const t = try b.declareArgs(.{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; } -fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization.?; - stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); - stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); - const w = linear.weight; - const scale = q.scales; - stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); - return scale; +const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; + +/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times +/// `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // Interleaving requires one binary instruction; SwiGLU chains several. + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); } -// Maximum total input tokens per MoE call before padding for the FP8 -// small-batch schedule. It loads each used expert's weights once and reuses -// them across the batch. -// Larger batches fall back to per-route or blockwise dispatch. -const fp8_small_batch_max_tokens = 64; - -fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { - return run(format, .small_batch, cfg, inputs); - } - if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { - return run(format, .per_route, cfg, inputs); - } - return run(format, .blockwise, cfg, inputs); +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// IO-bound tactics for activation stationary, compute-bound for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, blockwise: bool) void { + if (blockwise) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + .propagate_sparse_axis_from_op = "None", + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + .propagate_sparse_axis_from_op = "None", + }); } -fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - const call_cfg, const prepared = if (schedule == .small_batch) - prepareFp8SmallBatch(cfg, inputs) - else - .{ cfg, prepareRoutedInputs(cfg, inputs) }; +pub const ActivationStationary = tcl.Kernel(Cfg, .{ + .name = "moe_activation_stationary", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - const shape = prepared.x.shape(); - var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + var op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); + // Routing weights scale the activations, so the down projection + // also sums over the experts. + const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + }.run, +}); - if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); - return y.reshape(inputs.x.shape()); +/// Rows per expert block: the next power of two of the routes per expert. +pub fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); } -fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { - var prepared = inputs; - prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); - const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); - const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); - // These kernels gather every route, and blockwise also scatters each route - // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); - prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); - prepared.w = valid.select(w, Tensor.scalar(0, .f32)); - return prepared; -} +const blocks_per_chunk = 32; -fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); - const pad = padded.num_tokens - cfg.num_tokens; - var prepared = inputs; - prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); - prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - return .{ padded, prepared }; -} +pub const Blockwise = tcl.Kernel(Cfg, .{ + .name = "moe_blockwise", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); + const act = try gated(b, a, h, &.{ BGi, BB }, null); + op = b.tensorOperation(.{}); + const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, +}); test "tcl fused experts run on furiosa" { - try testFusedExperts(.{ .swiglu = .{} }); - try testFusedExperts(.geglu_tanh); -} - -fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // With two routes per token and eight experts, four tokens use per-route - // dispatch and sixteen use blockwise dispatch. + // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; @@ -171,15 +276,15 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { - return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = act, + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ + .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -231,12 +336,7 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - const gate = switch (activation) { - .swiglu => g / (1 + @exp(-g)), - .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), - else => unreachable, - }; - act[n] = gate * u; + act[n] = g / (1 + @exp(-g)) * u; } for (0..d) |m| { var y: f32 = 0; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index c74406c3..2bdebdfe 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,15 +5,12 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Activation = @import("api.zig").Activation; -const ExpertsLayout = @import("api.zig").ExpertsLayout; -const GlobalOptions = @import("api.zig").Options; -const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); +const Placement = @import("moe.zig").RoutingWeightPlacement; const Case = struct { backend: zml.moe.Backend, - activation: Activation = .{ .swiglu = .{} }, + activation: zml.moe.Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -25,22 +22,15 @@ const Case = struct { width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, - expert_mesh: *const zml.Mesh, - - fn inputShape(c: Case) Shape { - const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - return shape.withPartitioning(c.expert_mesh, spec); - } pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .fromShape(c.inputShape()); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .fromShape(c.inputShape()); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -62,7 +52,6 @@ const Case = struct { var output: zml.Bufferized(Outputs) = undefined; runner.run(io, .{input}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Outputs, &output); - try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. @@ -179,16 +168,20 @@ fn makeExpertFixture(c: Case) ExpertFixture { return fixture; } -fn interleave(tensor: Tensor, rows: i64) Tensor { - return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) - .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); +/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. +fn interleave(tensor: Tensor) Tensor { + const tags = tensor.shape().tags(); + return tensor.withTags(.{ .expert, .out, .in }) + .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) + .transpose(.{ .expert, .mid, .projection, .in }) + .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); } /// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> /// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); + return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) + .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); } fn packFp4Bytes(weight: Tensor) Tensor { @@ -196,11 +189,10 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, - .row_interleaved_1 => interleave(tensor, 1), - .row_interleaved_8 => interleave(tensor, 8), + .interleaved => interleave(tensor), }; } @@ -218,29 +210,17 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF } for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - switch (layout.packing) { - .swizzled_scales, .flashinfer_nvfp4 => { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; - }, - .mxfp4_interleaved8 => { - const q = &projection.quantization.?; - const sf = q.scales; - q.scales = sf.bitCast(.u8) - .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) - .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) - .withTags(sf.shape().transpose(.{ 0, 2, 1 })); - }, - .plain => {}, + if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; } switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); const q = &projection.quantization.?; + q.scales = q.scales.bitCast(.u8); if (scheme == .nvfp4) { - q.scales = q.scales.bitCast(.u8); // CuTe takes the input quantization multiplier and a GEMM // output multiplier that folds in its inverse. const input_scale = q.input_scale.?.asMultiplier(); @@ -283,26 +263,14 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: GlobalOptions = .{ + const options: zml.moe.Options = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, }; - const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ - .in_specs = .{ - .input = data_spec, - .ids = data_spec, - .weights = data_spec, - .gate_up = expert_spec, - .down = expert_spec, - }, - .out_specs = data_spec, - }, options); - // Keep the oracle input replicated so token sharding does not change its GEMM rounding. - const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); + const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -317,7 +285,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]Activation{ +const gated_activations = [_]zml.moe.Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -327,8 +295,8 @@ const gated_activations = [_]Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]Activation{ +const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]zml.moe.Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -366,7 +334,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const Activation, + activations: []const zml.moe.Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, @@ -378,7 +346,7 @@ const Matrix = struct { } /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { std.debug.assert(index < self.count()); var remaining = index; const shape = self.shapes[remaining % self.shapes.len]; @@ -406,7 +374,6 @@ const Matrix = struct { .width = shape.width, .intermediate = shape.intermediate, .experts = shape.experts, - .expert_mesh = expert_mesh, }; } }; @@ -447,7 +414,7 @@ const Pipeline = struct { var index: usize = 0; for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); + const c = matrix.caseAt(backend, local_index); if (available) { case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { error.Closed => {}, @@ -624,7 +591,6 @@ test "Triton MoE prefill launch buckets fit shared memory" { for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { const case: Case = .{ .backend = .triton, - .expert_mesh = platform.replicated_mesh, .tokens = tokens, .width = 256, .intermediate = 128, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 6e4006fa..3eb457da 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,6 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1027,7 +1026,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1093,7 +1092,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 8ba799a5..908cd163 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,14 +6,10 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; const kernels = @import("moe.zig"); +const toDType = zml.kernel.triton.from; +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton"); @@ -39,38 +35,109 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: LocalRouting, + ids: zml.Tensor, + weights: zml.Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: Options, + options: zml.moe.Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. if (gate_up.quantization) |q| switch (q.scheme) { .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, routing, gate_up, down, options); + return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); }, else => {}, }; - const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; - return fusedExpertsBase(input, routing, gate_up, down, layout, options); + return fusedExpertsBase(input, ids, weights, gate_up, down, options); } pub fn fusedExpertsBase( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - layout: ProjectionLayout, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -87,10 +154,10 @@ pub fn fusedExpertsBase( }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -118,13 +185,13 @@ pub fn fusedExpertsBase( stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| + if (opts.expert_map) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (expert_map_) |expert_map| + const expert_ids = if (opts.expert_map) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -134,9 +201,9 @@ pub fn fusedExpertsBase( const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = gate_up_.bias, + .bias = opts.gate_up.bias, .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), + .weight_scale = opts.gate_up.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -145,7 +212,7 @@ pub fn fusedExpertsBase( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = opts.activation.applyActivation(gate_up_out, layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -155,9 +222,9 @@ pub fn fusedExpertsBase( const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = down_.bias, + .bias = opts.down.bias, .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), + .weight_scale = opts.down.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -494,14 +561,14 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { - return activation.applyActivation(x, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { + return zml.moe.applyActivation(x, activation, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); @@ -510,7 +577,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -666,8 +733,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. @@ -699,12 +765,13 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsBase(x, .{ - .ids = route.addConstant(2), - .weights = route.convert(.f32).addConstant(1).scale(0.25), - .global_num_experts = 8, - .local_num_experts = 8, - }, gate_up, down, layout, .{ + return fusedExpertsImpl(.{ + .hidden_states = x, + .gate_up = gate_up, + .down = down, + .topk_ids = route.addConstant(2), + .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), + .gate_up_layout = layout, .routing_weight_placement = placement, .quantize_input = false, .activation = .{ .swiglu = .{} }, @@ -719,7 +786,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); @@ -799,13 +866,58 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: Options) void { +pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); } +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); + + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight.bitCast(.u8), + .s1 = gq.scales, + .w2 = down.weight.bitCast(.u8), + .s2 = dq.scales, + .global_experts = gate_up.weight.dim(.expert), + .topk = ids.dim(.topk), + .limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + .expert_parallel = expert_parallelism, + }; + + return if (expert_parallelism) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); +} + pub const Config = struct { tokens: i64, hidden: i64, @@ -840,233 +952,210 @@ pub const Config = struct { } }; -fn validateInputs( +pub fn validateInputs( c: Config, input: zml.Tensor, ids: zml.Tensor, - expert_weights: zml.Tensor, - gate_up: zml.Tensor, - gate_up_scales: zml.Tensor, - down: zml.Tensor, - down_scales: zml.Tensor, -) void { - stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); - + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, +) !void { + if (s1.rank() != 3 or s2.rank() != 3) return error.InvalidInputShape; + if (s1.dim(2) < @divTrunc(c.hidden, 32) or + s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; const shapes = .{ - .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), - .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), - .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), - .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), + .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), + .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, s1.dim(2) }, .f8e8m0), + .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), + .s2 = zml.Shape.init(.{ c.experts, c.hidden, s2.dim(2) }, .f8e8m0), }; - stdx.debug.assert(input.shape().eql(shapes.input) and - ids.shape().eql(shapes.ids) and - expert_weights.shape().eql(shapes.expert_weights) and - // Note: we accept both i8 and u8 for gate_up and down - // TODO: we should instead use f4e2m1 - (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and - gate_up_scales.shape().eql(shapes.gate_up_scales) and - (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and - down_scales.shape().eql(shapes.down_scales), - \\Invalid MXFP4 tensor layout: - \\ - expected input={f}, got {f} - \\ - expected ids={f}, got {f} - \\ - expected expert_weights={f}, got {f} - \\ - expected gate_up={f}, got {f} - \\ - expected gate_up_scales={f}, got {f} - \\ - expected down={f}, got {f} - \\ - expected down_scales={f}, got {f} - , .{ - shapes.input, input, // - shapes.ids, ids, // - shapes.expert_weights, expert_weights, // - shapes.gate_up, gate_up, // - shapes.gate_up_scales, gate_up_scales, // - shapes.down, down, // - shapes.down_scales, down_scales, // - }); - - stdx.debug.assert( - gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and - down_scales.dim(2) >= @divTrunc(c.intermediate, 32), - "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", - .{ c, gate_up_scales, down_scales }, - ); + if (!input.shape().eql(shapes.x)) return error.InvalidInputShape; + if (!ids.shape().eql(shapes.ids)) return error.InvalidInputShape; + if (!weights.shape().eql(shapes.scales)) return error.InvalidInputShape; + if (!w1.shape().eql(shapes.w1)) return error.InvalidInputShape; + if (!s1.shape().eql(shapes.s1)) return error.InvalidInputShape; + if (!w2.shape().eql(shapes.w2)) return error.InvalidInputShape; + if (!s2.shape().eql(shapes.s2)) return error.InvalidInputShape; } -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( +const Context = struct { input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + global_experts: i64, + topk: i64, + limit: f32, + routing_weight_placement: RoutingWeightPlacement, + expert_parallel: bool, - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); + const tokens = ids.dim(.token); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + ids = ids.sub(partition_id.scale(experts)); + } + const x = self.input.reshape(.{ tokens, hidden }); + const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = self.global_experts, + .topk = self.topk, + .swiglu_limit = self.limit, + .routing_weight_placement = self.routing_weight_placement, + }; - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, self.w1, self.s1, self.w2, self.s2) catch @panic("Invalid MXFP4 tensor layout"); - const x = input.reshape(.{ tokens, hidden }); - const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + const bn = 16; + const bk = 512; - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = routing.global_num_experts, - .topk = topk, - .swiglu_limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - }; + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); - - const bn = 16; - const bk = 512; - - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); - - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, - .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, .num_warps = 4, .num_stages = 1, }, ); - map = s.map; - perm = s.perm; - sched = s.sched; + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = self.w1, .ws = self.s1.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.s1.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = self.w2, .ws = self.s2.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.s2.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + const result = kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = gq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = dq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - return kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(input.shape().dims()).withTags(input.shape()); -} +}; diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig deleted file mode 100644 index 0e288856..00000000 --- a/zml/kernels/moe/weights.zig +++ /dev/null @@ -1,10 +0,0 @@ -const nn = @import("../../nn.zig"); -const Tensor = @import("../../tensor.zig").Tensor; - -pub fn unpackedWeight(linear: nn.Linear) Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 587af69f..9fd6a355 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -54,7 +54,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ + const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -115,31 +115,6 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { - const Call = struct { - inputs: Inputs, - case: cases.Case, - - fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forward(self.inputs, self.case); - } - }; - const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ - .in_specs = .{ .inputs = .{ - .q = heads, - .k = heads, - .v = heads, - .table = .replicated, - .lengths = .replicated, - .starts = .replicated, - .sinks = .parse(.{ ._0 = .model }), - .indices = .replicated, - } }, - .out_specs = heads, - }); -} - fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ef78cc49..e572cce1 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,10 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -pub const AttentionOptions = @import("../options.zig").AttentionOptions; -const kernel = @import("attention.zig"); +const paged = @import("../paged_attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); +const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -64,7 +64,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); @@ -72,44 +72,57 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const indices = switch (opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), - }; - const cfg: kernel.Config = .{ - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages = k_.dim(.page), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .dtype = zml.kernel.cute.from(q_.dtype()), - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), - .sink = opts.sink != null, - .indexed = opts.mask == .indices, - .topk = indices.dim(-1), - }; - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - return kernel.Kernel.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, - .block = .{ 32, 1, 1 }, - }).out.transpose(q.shape()); + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); } + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const indices = switch (self.opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages = k.dim(.page), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q.dtype()), + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), + .sink = self.opts.sink != null, + .indexed = self.opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q, + .k = k, + .v = v, + .table = p.block_table, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(output_shape); + } +}; diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index c0815d1d..8c289b04 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,7 +1,5 @@ const std = @import("std"); - const zml = @import("zml"); - const cases = @import("cases.zig"); pub const Fixture = struct { @@ -124,7 +122,7 @@ pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { }, }, }; - return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); + return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); } pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 12d6273c..89761bdd 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -317,7 +317,7 @@ fn produce(t: Tile) void { const id = tiles.iv.mul(c.features() * 2).add(stage); var reuse = b.openIf(id.ge(c.buffers())); { - b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); reuse.close(); } const source = if (stage < c.features()) t.k else t.v; @@ -339,7 +339,7 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { { for (0..@as(usize, @intCast(c.features()))) |part| { const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); - b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); + b.mbarrierWait(t.ready(id), t.phase(id)); b.tcgen05Fence(.after); const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); @@ -353,22 +353,22 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); } - b.tcgen05Commit(t.released(id), .{}); + b.tcgen05Commit(t.released(id)); } - b.tcgen05Commit(t.shared.add(c.completion()), .{}); + b.tcgen05Commit(t.shared.add(c.completion())); leader.close(); } - b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1)); b.tcgen05Fence(.after); // A fast warp must not wrap completion parity before every consumer has // observed the preceding PV completion. - if (!qk) b.barrierCtaSyncAligned(1, 128); + if (!qk) b.barrierSync(1, 128); } fn publishProbabilities(t: Tile) void { t.b.fenceProxyAsyncShared(); t.b.tcgen05Fence(.before); - t.b.barrierCtaSyncAligned(1, 128); + t.b.barrierSync(1, 128); t.b.tcgen05Fence(.after); } @@ -393,7 +393,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu leader.close(); } } - b.barrierCtaSyncAligned(1, 128); + b.barrierSync(1, 128); var result: [heads]V = undefined; for (0..heads) |h| { var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); @@ -403,7 +403,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu } result[h] = value; } - b.barrierCtaSyncAligned(1, 128); + b.barrierSync(1, 128); return result; } @@ -514,7 +514,7 @@ fn prefill(t: Tile) void { preserve.yieldElse(@as([64]V, @splat(zero))); } @memcpy(&saved, &preserve.results); - b.barrierCtaSyncAligned(1, 128); + b.barrierSync(1, 128); } multiply(t, tiles.iv, true); var scores: [64]V = undefined; diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index afc4298e..664720ee 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,6 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,48 +19,61 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const decode = !parameters.options_.is_prefill; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; - var c: kernel.Config = .{ - .architecture = if (cc.minor >= 3) .sm103 else .sm100, - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q_.dtype()), - .decode = decode, - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), - .sink = opts.sink != null, - }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); - // An unused scalar stays a constant buffer; broadcasting a dummy for - // every head would add a helper kernel to each attention invocation. - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ - .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(q.shape()); +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); } +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const decode = !p.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q.dtype()), + .decode = decode, + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + .sink = self.opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = p.block_table, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(output_shape); + } +}; + const ArchitectureCheck = struct { fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6645abf1..6dfeb7e2 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,6 +1,7 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -17,53 +18,66 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const decode = !parameters.options_.is_prefill; - var c: kernel.Config = .{ - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q_.dtype()), - .warps_per_query = if (decode) 4 else 2, - .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), - .sink = opts.sink != null, - }; - // Mixed batches also contain single-token rows. Partition their KV - // range so one long decode cannot hold up all the prefill CTAs. - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); - const out = kernel.Kernel.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, - .block = .{ 128, 1, 1 }, - }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(q.shape()); +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); } + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const decode = !p.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q.dim(.hd) == 512) 32 else 64, + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + .sink = self.opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q, + .k = k, + .v = v, + .table = p.block_table, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(output_shape); + } +}; diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 8ccc6fd9..5624de0e 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -10,13 +10,11 @@ transposes queries and caches into the kernels' layouts, launches the the output. Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which -is `bf16` only: a threadgroup holds up to four query heads of a KV head, times -splits of the KV positions, at most 256 threads. Rows split over the positions -are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count -and the number of rows. Everything else, `f16` decode included, runs -`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of -8, 16 or 32 positions), then `zero_padding_rows`. +decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` +only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head +dimension 512 past 4096 KV positions. Everything else, `f16` decode included, +runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, +pages of 8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale are passed to the kernels. Explicit key indices are rejected, and diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index 2cec68f5..e2d68398 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,7 +3,8 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; + +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); @@ -85,8 +86,8 @@ const DispatchOptions = struct { scale: f32, }; -/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled -/// otherwise. +/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at +/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. /// `q` and the result are `[tokens, heads, head_dim]`, the caches /// `[pages, block_size, kv_heads, head_dim]`, `block_table` /// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` @@ -105,7 +106,7 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table const max_blocks: i32 = @intCast(block_table.dim(1)); const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and + (head_dim == 128 or head_dim == 256 or head_dim == 512) and dtype == .bf16; if (vec_decode) { @@ -128,30 +129,35 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table }; const kv_capacity = max_blocks * block_size; - const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); - const nrows = total_q_tokens * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); + if (head_dim == 512 and kv_capacity > 4096) { + const nwg = 4; + const nrows = total_q_tokens * num_heads; - const cfg: paged_attn.FaVecPaged.Cfg = .{ - .head_dim = head_dim, - .block_size = block_size, - .position_stride = num_kv_heads * head_dim, - .heads = launch.heads, - .splits = launch.splits, - }; - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; + const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, + .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, 8, 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), + .constants = .{ .args = .of(args) }, + }).dst; - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ + .cfg = .{ .nwg = nwg, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), 1, 1 }, + .threadgroup = .{ 32, nwg, 1 }, + .constants = .{ .args = .of(@as(i32, nrows)) }, + }).dst; + } + + var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; + if (head_dim == 512 and idx > 1) idx = 1; + const nsg = ([_]i32{ 4, 8, 16 })[idx]; + + return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, + .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(nsg), 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), .constants = .{ .args = .of(args) }, }).dst; } @@ -205,36 +211,3 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 04f4c1ac..20429565 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,8 +3,9 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const kernel = @import("../../../kernel.zig"); +const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; @@ -17,18 +18,25 @@ const masked_score = -flt_max / 2.0; const max_half = 65504.0; const simd_width = 32; -/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. +fn pad2(x: i32, n: i32) i32 { + return (x + n - 1) & ~(n - 1); +} + +/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup +/// of `nsg` simdgroups per (position split, query head, query row). pub const FaVecPaged = struct { pub const Cfg = struct { - /// The key and value head dimension. + /// The key and value head dimension, DK = DV. head_dim: i32, + /// Simdgroups splitting the KV positions. + nsg: i32, /// The page size. block_size: i32, /// The in-page position stride, n_kv * head_dim elements. position_stride: i32, - heads: i32, - splits: i32, + /// Position-split threadgroups. Above 1 the kernel writes f32 partials + /// for FaVecPagedReduce. + nwg: i32, }; pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ @@ -58,10 +66,17 @@ pub const FaVecPaged = struct { .run = run, }); + /// The threadgroup memory of a launch with `nsg` simdgroups. + pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { + const padded = pad2(head_dim, 128); + const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; + return @intCast(@divFloor(bytes + 15, 16) * 16); + } + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), } const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; @@ -74,90 +89,158 @@ pub const FaVecPaged = struct { .seq_lens = int_in, .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, + .shmem_f16 = .{ .threadgroup = .half }, .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, .tgpg = .{ .threadgroups_per_grid = .uint3 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, }); impl(b, cfg, a); } + /// The attention body. fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; + const DK = cfg.head_dim; + const DV = cfg.head_dim; + const NE = 1; + const C = 32; + const NSG = cfg.nsg; const BS = cfg.block_size; const PS = cfg.position_stride; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; + const NWG = cfg.nwg; - const num_heads = a.args.field("num_heads"); - const group = num_heads.div(a.args.field("n_kv")); - const groups = group.div(HG); const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + const iq2 = a.tgpig.at(1).to(.ushort); + const iwg = a.tgpig.at(0).to(.short); + + const DK4 = @divExact(DK, 4); + const DV4 = @divExact(DV, 4); + + const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); - // Padding rows: FaVecPagedReduce zeroes them when there are splits. var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); - var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); - { - dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); - loop.close(); + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + const zero = b.constant(.float4, 0).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), zero); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); + lane0.close(); + } } + first.close(); } b.return_(); padding.close(); } - const seq = sequenceOf(b, a, r); - const kv_len = seq.kv_len; - const kv_lo = seq.kv_lo; - const ic0_lo = kv_lo.div(C); - const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); - const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const seq_idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); + const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); + const kv_total = a.seq_lens.load(seq_idx); + const kv_len = kv_total.sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + + const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + + const PK = pad2(DK, 128); + const PK4 = @divExact(PK, 4); + const PV = pad2(DV, 128); + const PV4 = @divExact(PV, 4); + const NW = simd_width; + const NL = @divExact(NW, NE); + const SH = 4 * C; + + const shmem = a.shmem_f16; + const sq4 = shmem.add(0 * PK).to(.half4); + const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); + const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); + const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); + const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); + // This lane's output, until the simdgroups merge theirs. + const lane_so4 = so4.add(a.tiisg); + + const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); + const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); + const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + + const q4 = q.to(.bfloat4); + var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + { + const i = load_q.iv; + var value = b.openIfElse(i.lt(DK4), .{.half4}); + { + value.yieldThen(.{q4.load(i).to(.half4)}); + } + { + value.yieldElse(.{b.constant(.half4, 0)}); + } + sq4.store(i, value.results[0]); + load_q.close(); + } + var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); + { + lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); + zero_o.close(); + } + var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); + { + ss4.store(zero_s.iv, b.constant(.float4, 0)); + zero_s.close(); } + b.threadgroupBarrier(.mem_threadgroup); + + // When DV4/NL is a multiple of NW every lane updates the output, so the + // `ty == 0` guard is dropped at comptime. + const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; + const full: metal.LoopOpts = .{ .unroll = .full }; const S = b.local(.float, 0.0); const M = b.local(.float, -flt_max / 2.0); - const ic0 = b.local(.int, ic0_lo.add(iwg)); + const tx = a.tiisg.rem(NL).to(.short); + const ty = a.tiisg.div(NL).to(.short); + + const ic0_lo = kv_lo.div(C); + const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); var blocks = b.openLoop(.{}); { const ic = ic0.mul(C); @@ -167,103 +250,175 @@ pub const FaVecPaged = struct { past.close(); } - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); + sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); + + const pq4 = sq4.add(tx); + const mqk = b.zeroedArray(.float, C / NE); + var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); { const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); { const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); + mqk.store(cc, mqk.load(cc).add(dot)); dots.close(); } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } + mqk.store(cc, b.simdSum(mqk.load(cc))); qk.close(); } - const pos = ic.add(a.tiisg.to(.int)); - const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); - const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); + ss.store(b.lift(NE).mul(tx).add(ty), score); + + b.simdgroupBarrier(.mem_threadgroup); const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); + const s = ss.load(a.tiisg); + + M.assign(b.simdMax(b.max(M.rvalue(), s))); + const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); + const vs = b.exp(s.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + ss.store(a.tiisg, vs); + + var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + const at = rescale.iv.mul(NL); + lane_so4.store(at, lane_so4.load(at).mul(ms)); rescale.close(); } + if (rescale_lanes) |*l| l.close(); - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + b.simdgroupBarrier(.mem_threadgroup); + + const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); + var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + lo.store(zero_lo.iv, b.constant(.float4, 0)); + zero_lo.close(); + } + + const sst = ss.add(ty); + + var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); { const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); { const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); + lo.store(ii, lo.load(ii).add(weighted)); dims.close(); } pv.close(); } + var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + const ii = add_o.iv; + const partial = lo.load(ii); + lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); + add_o.close(); + } + if (add_lanes) |*l| l.close(); blocks.next(); - ic0.assign(ic0.add(live)); + ic0.assign(ic0.add(NWG * NSG)); blocks.close(); } - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } + var stats = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S); + ss.store(1, M); + stats.close(); } - var first = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + + b.threadgroupBarrier(.mem_threadgroup); + + const rr = b.local(.short, @divFloor(NSG, 2)); + var halving = b.openLoop(.{}); + { + halving.body(rr.gt(0)); + } + { + var group = b.openIf(a.sgitg.lt(rr)); { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); + const S0 = ss.load(0); + const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); + + const M0 = ss.load(1); + const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); + + const M_ = b.max(M0, M1); + + const ms0 = b.exp(M0.sub(M_)); + const ms1 = b.exp(M1.sub(M_)); + + const S_ = S0.mul(ms0).add(S1.mul(ms1)); + + var lane0 = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S_); + ss.store(1, M_); + lane0.close(); + } + + var merge = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const i = merge.iv; + so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); + merge.close(); + } + group.close(); } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); + b.threadgroupBarrier(.mem_threadgroup); + rr.assign(rr.shr(1)); + halving.close(); + } + + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), out); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const part = so4.load(loop.iv); + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const sum = ss.load(0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); + const max = ss.load(1); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); + lane0.close(); + } } + first.close(); } - if (first) |*scope| scope.close(); } /// The address of position `p`'s bf16 row in `cache`, through the page @@ -275,144 +430,85 @@ pub const FaVecPaged = struct { } }; -/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). -fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); - { - first_seq.assign(mid); - half.elseBranch(); - } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); - } - const idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(idx)); - const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); - const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; -} - -fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { - const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); - return b.min(splits, chunks); -} - -/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. +/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, +/// one threadgroup per row and head. pub const FaVecPagedReduce = struct { pub const Cfg = struct { - splits: i32, + /// The partials to merge, FaVecPaged's `nwg`. + nwg: i32, /// The head dimension. head_dim: i32, }; - pub const simdgroups = 8; + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ + .{ .name = "nrows", .type = .int }, + } }; pub const Kernel = metal.Kernel(Cfg, .{ .name = "fa_vec_paged_reduce", - .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, + .inputs = &.{"htmp"}, .outputs = &.{"dst"}, .constants = &.{"args"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVecPaged.Args }, + .args = .{ .constant_struct = Args }, .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .seq_lens = int_in, - .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tgpig = .{ .threadgroup_position_in_grid = .uint }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); + const nwg = cfg.nwg; const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; + const rid = a.tgpig.to(.ulong); + const iwg = a.tiisg.to(.short); + const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - - const live = b.local(.int, 0); - var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + var max = b.openIfElse(iwg.lt(nwg), .{.float}); { - live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); - real.close(); + max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); } - var empty = b.openIf(live.le(0)); { - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); - lane0.close(); - } - b.return_(); - empty.close(); + max.yieldElse(.{-flt_max / 2.0}); + } + const M = max.results[0]; + var sum = b.openIfElse(iwg.lt(nwg), .{.float}); + { + sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); + } + { + sum.yieldElse(.{0.0}); } - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); + const m = b.simdMax(M); + const ms = b.exp(M.sub(m)); + const S = b.simdSum(sum.results[0].mul(ms)); const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + const DV4 = @divTrunc(DV, 4); + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); + const i = loop.iv; + var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); + part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); + { + part_.yieldElse(.{b.constant(.float4, 0)}); + } + const part = part_.results[0]; + const v = b.simdSum(part); + var lane0 = b.openIf(iwg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + loop.close(); } } }; @@ -888,14 +984,13 @@ pub const PagedAttentionTiled = struct { test "the paged attention kernels emit verified modules" { const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + for ([_]i32{ 128, 256, 512 }) |head_dim| { + for ([_]i32{ 4, 8, 16 }) |nsg| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); } } + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); for ([_]Type{ .half, .bfloat }) |dtype| { allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 84716379..ef9e852c 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -153,28 +153,32 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", .{ q.dim(.hd), target_head_dim }, ); - return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); + var q_padded = q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); + q_padded._shape._sharding = q._shape._sharding; + return q_padded; } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); + const mesh = zml.Compiler.current().resolveMesh(.{.model}); + const kv_pages_partitioned = kv_pages.withPartitioning(mesh, .{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(mesh, .{ .h = .model }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); - const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); stdx.debug.assert( cache_head_dim == alignedKernelHeadDim(logical_head_dim), "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", .{ cache_head_dim, logical_head_dim }, ); - const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); - const query_start_len = parameters.query_start_len; + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); + const query_start_len = parameters.query_start_len.withPartitioning(mesh, .{ .b = .replicated }); return .{ .q = q_ragged, - .kv_pages = kv_pages, - .seq_lens = parameters.seq_lens, - .block_table = parameters.block_table, + .kv_pages = kv_pages_partitioned, + .seq_lens = parameters.seq_lens.withPartitioning(mesh, .{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(mesh, .{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len), + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(mesh, .{ ._0 = .replicated }), }; } @@ -220,9 +224,57 @@ pub const mosaic_tpu = struct { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); + const prepared = prepareInputs(parameters, q, kv_cache); - const cfg = buildCfg(prepared, parameters, opts); - const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); + + const q_out = zml.ops.manualComputation( + (struct { + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + opts: AttentionOptions, + parameters: Parameters, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + const prepared_inputs: PreparedInputs = .{ + .q = self.q, + .kv_pages = self.kv_pages, + .seq_lens = self.seq_lens, + .block_table = self.block_table, + .query_start_len = self.query_start_len, + .num_seqs = self.num_seqs, + }; + + const cfg = buildCfg(prepared_inputs, self.parameters, self.opts); + const q_out = raggedPagedKernelCall( + prepared_inputs.q, + prepared_inputs.kv_pages, + prepared_inputs.seq_lens, + prepared_inputs.block_table, + prepared_inputs.query_start_len, + prepared_inputs.num_seqs, + cfg, + ); + stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); + return q_out; + } + }).body, + .{ + .q = prepared.q, + .kv_pages = prepared.kv_pages, + .seq_lens = prepared.seq_lens, + .block_table = prepared.block_table, + .query_start_len = prepared.query_start_len, + .num_seqs = prepared.num_seqs, + .opts = opts, + .parameters = parameters, + }, + prepared.q.shape(), + ); + const restored = restoreQueryHeads(q, q_out); stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); return restored; diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig deleted file mode 100644 index 35ad4049..00000000 --- a/zml/kernels/paged_attention/options.zig +++ /dev/null @@ -1,36 +0,0 @@ -const Tensor = @import("../../tensor.zig").Tensor; - -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?Tensor = null, -}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 4a97a38e..99065140 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,14 +3,12 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); -pub const AttentionMask = @import("options.zig").AttentionMask; -pub const AttentionOptions = @import("options.zig").AttentionOptions; -pub const cute = @import("cute/cute.zig"); +const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); const triton = @import("triton/triton.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const stablehlo = @import("stablehlo/stablehlo.zig"); const PagedAttention = @This(); @@ -32,7 +30,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { - .cuda => if (cute.isAvailable(platform)) .cute else .triton, + .cuda => .triton, .rocm => .triton, .oneapi => .triton, .tpu => .mosaic_tpu, @@ -153,6 +151,41 @@ pub const Parameters = union(Backend) { } }; +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: zml.Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -199,53 +232,49 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } -}; -pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - _ = k; - _ = v; - const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); -} - -const Attention = struct { - parameters: Parameters, - q: zml.Tensor, - kv_cache: KvCache, - opts: AttentionOptions, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - return switch (args.parameters) { - .cute => |p| switch (args.kv_cache) { - .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), - .dense => @panic("CuTe paged attention requires split KV pages"), - }, - .triton => |triton_parameters| switch (args.kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), - }, - .metal => |metal_parameters| switch (args.kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .stablehlo => |params| switch (args.kv_cache) { - .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .tcl => |params| switch (args.kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + pub fn getPage( + self: KvCache, + page_index: zml.Tensor, + ) [2]zml.Tensor { + return switch (self) { + .split => |split| .{ + split.k.slice(split.k.axis(.page), .dynSingle(page_index)), + split.v.slice(split.k.axis(.page), .dynSingle(page_index)), }, + .dense => @panic("TODO"), }; } }; +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { + _ = k; + _ = v; + return switch (parameters) { + .cute => |p| switch (kv_cache) { + .split => |split| cute.pagedAttention(p, q, split.k, split.v, opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), + }, + .metal => |metal_parameters| switch (kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), + .tcl => |params| switch (kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, q, split.k, split.v, opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, + }; +} + test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ .arena = undefined, @@ -306,22 +335,21 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; - const tp = zml.testing.tpMesh(); - const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); + const model_meshe = platform.meshes.get("model").?; const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), .kv_cache = .{ .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), }, }, }; - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); @@ -434,22 +462,11 @@ test pagedAttention { backend_options_args.backend = backend; const parameters: Parameters = .init(.fromBackend(backend_options_args)); - const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const partitioning: Partitioning = .{ - .in_specs = .{ - .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), - .q = head_spec, - .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, - .opts = .{ .mask = .{ .standard = {} }, .sink = null }, - }, - .out_specs = head_spec, - }; - const exe = try platform.compileFn( allocator, io, pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 55ee01d5..ab0ffee2 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,12 @@ const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const KvCache = @import("../paged_attention.zig").KvCache; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, + kv_cache: KvCache, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,12 +14,11 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = k.dim(.k_chunk); - const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); + const page_size = kv_cache.split.k.dim(.k_chunk); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, + .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -29,17 +28,16 @@ pub fn pagedAttention( // Note: we do all partial softmax in f32 .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(padded_q.shape()), + .out = .zeroes(q.shape()), }, ); - return final_state.out.slice(.b, .{ .end = q.dim(.b) }); + return final_state.out; } const AttentionLoop = struct { q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, + kv_cache: KvCache, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -67,7 +65,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -95,7 +93,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); } /// Decode version: num_slots == 1, we multiply one query with a page of keys. @@ -114,7 +112,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); } }, .{ @@ -135,10 +133,7 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - - const page_axis = self.k.axis(.page); - const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); - const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); + const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { @@ -167,11 +162,7 @@ const AttentionLoop = struct { return og_softmax.merge(partial); } - fn nextQueryStart(self: While, state: State) zml.Tensor { - return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); - } - - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { return zml.ops.if2( query_has_more_keys, // If the query has more pages: update partial softmax with attention on current page, delay updating output. @@ -190,7 +181,7 @@ const AttentionLoop = struct { .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), + .slot_id = state.slot_id.addConstant(num_slots), // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index d04b684e..ce8bdb4f 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1,55 +1,42 @@ -//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. -//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. -//! `Repro:` files are in the reproducer repository, platform/furiosa. +//! Furiosa RNGD paged attention in TCL, using the common/flash attention +//! algorithms from Furiosa's SDK (`furiosa-kernels 2026.4.0` reference). +//! This implementation retains FP32 normalization/accumulation and BF16 +//! contractions. The exact common SDK operation materializes scores in BF16; +//! it fails the required unit-scale tolerance and is not a selectable backend. +//! +//! Queries are cut in blocks of rows of one sequence (one token per block for +//! decode). A context of at most `chunk_keys` keys runs Furiosa's one-shot +//! attention: gather the keys by token slot, contract, mask, one softmax, +//! contract. Longer contexts run Furiosa's flash attention: a device loop over +//! chunks of `chunk_keys` keys up to the longest sequence, gathering whole +//! pages and accumulating the softmax statistics in f32. +//! +//! Decode without sinks or windows, up to `decode_keys` keys, instead runs one +//! fusible kernel per layer over the whole pages each token sees, with the +//! token metadata every layer shares computed once per forward pass. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); +const builder = @import("dsl/tcl/builder"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const AttentionOptions = @import("../options.zig").AttentionOptions; +const paged_attention = @import("../paged_attention.zig"); const triton = @import("../triton/triton.zig"); -/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to -/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. -/// Remove once 8192-key chunks compile. const chunk_keys = 4096; -const flash_chunk_pairs = 512 * 2048; -const prefill_chunk_keys = 2048; const prefill_block = 32; -/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout -/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. -/// Remove once any count compiles. -fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { - const blocks = @divTrunc(tokens + sequences * (block - 1), block); - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); -} - -fn prefillBlock(tokens: i64, sequences: i64) i64 { - const short = prefill_block / 2; - return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; -} - -/// A straight-line head over each token's first `head` keys, then a device loop over -/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. -pub const Plan = struct { - head: i64, - chunk: i64, - group: i64, -}; - -pub var calibration_plan: ?Plan = null; - pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. - /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches + /// `{.page, .k_chunk, .hkv, .hd}`. Sinks are one logit per query head. + /// Unwritten cache entries are excluded before the contractions; callers + /// need only populate the tokens covered by their sequence lengths. + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -59,7 +46,7 @@ pub const paged = struct { stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); const compiler = zml.Compiler.current(); - stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); + stdx.debug.assert(compiler.platform.target == .furiosa and compiler.meshes[0].numDevices() == 1, "TCL attention requires one Furiosa chip", .{}); const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); defer attributes.deinit(); @@ -67,33 +54,25 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - return attend(parameters, q, k_cache, v_cache, opts); - } - - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); - // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to - // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. - // Remove once 16- to 48-key decode compiles. + // TCC 2026.5 has no DPE tactic for 32-key attention. Pad to + // at least 64 keys; the extra pages remain invalid and contribute zero. const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; - // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 - // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: - // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. + // Chunks hold whole pages and a multiple of 64 keys: TCC 2026.5 fails + // to allocate a chunk of 4080 keys (85 pages of 48). const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; const alignment = if (key_alignment <= chunk_keys) key_alignment else page; + const chunk = if (keys <= chunk_keys) keys else @min(keys, @max(1, @divTrunc(chunk_keys, alignment)) * alignment); const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); - const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); - const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); - const rows = block_count * block; - // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. - const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; - const chunk = @min(keys, chunk_limit); - var cfg: Cfg = .{ + const blocks = (std.math.divCeil(i64, q.dim(.b), prefill_block) catch unreachable) + parameters.seq_lens.dim(.b); + // Regular block capacities limit layout-search cost for the fused graph. + // Extra blocks retain zero valid lengths and invalid query positions. + const cfg: Cfg = .{ .tokens = q.dim(.b), .sequences = parameters.seq_lens.dim(.b), - .b = block_count, - .i = block, + .b = if (decode) q.dim(.b) else @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable), + .i = if (decode) 1 else prefill_block, .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, .chunk = chunk, .hkv = q.dim(.hkv), @@ -107,24 +86,10 @@ pub const paged = struct { .is_causal = opts.mask.isCausal(), .window = @intCast(opts.mask.slidingWindow() orelse 0), }; - const plain_decode = decode and !cfg.use_sink and cfg.window == 0; - // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens - // over a non-power-of-two page count fail to compile, so straight decode reads - // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, - // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. - const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; - const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; - const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); - if (plan.head > 0 and plan.chunk > 0) { - stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); - return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); - } - if (plan.chunk == 0 and straight) { - return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ + if (decode and !cfg.use_sink and cfg.window == 0 and keys <= decode_keys) { + return decodeAttention(parameters, q, k_cache, v_cache, .{ .t = cfg.tokens, - .a = straight_keys, - .group = plan.group, + .a = keys, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, @@ -132,90 +97,34 @@ pub const paged = struct { .pg = page, .scale = cfg.scale, .is_causal = cfg.is_causal, - .window = cfg.window, - .use_sink = cfg.use_sink, }).convert(q.dtype()); } - cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; - cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); - // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size - // other than a power of two fails ("Inconsistent tensor group - // offsets"): pad the groups with zero queries, whose outputs are - // dropped. Repro: - // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, - // platform/furiosa). Remove once any group size compiles. - const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); - const extra = groups - cfg.g; - cfg.g = groups; - const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); const inputs: OneShot.Inputs = .{ - .q = queries, + .q = q.convert(.bf16), .pages = table, .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), + .sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)), }; - const outputs: OneShot.Outputs = .{ .out = queries.shape() }; + const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; const out = if (cfg.a == cfg.chunk) OneShot.call(inputs, outputs, .{ .cfg = cfg }).out else Flash.call(inputs, outputs, .{ .cfg = cfg }).out; - return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); + return out.convert(q.dtype()); } }; -fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { - // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in - // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. - // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. - if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { - if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; - return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; - } - if (straight and cfg.window > 0) { - const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); - const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; - if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) - return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; - } - if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) - return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; - if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; - return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; -} - -const window_chunk_keys = 1024; -const window_flash_chunks = 16; -const batch_window_flash_chunks = 4; - -/// Widest block table the straight-line decode kernel takes for one token. -/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot -/// propagate gather bounds outside a loop over a token axis. Repro: -/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. -const straight_decode_keys = 32768; - -/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC -/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. -/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. +/// Keys a decode kernel attends to in one pass: TCC 2026.5 fails on the +/// decode metadata for 131072 keys in pages of 16. const decode_keys = 65536; -const chunked_decode_tokens = 16; -const straight_batch_decode_keys = 8192; - -/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero -/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more -/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. -const unmasked_decode_keys = 1 << 17; - -const masked_decode_keys = 4096; - const DecodeCfg = struct { t: i64, a: i64, - group: i64, hkv: i64, g: i64, d: i64, @@ -223,12 +132,14 @@ const DecodeCfg = struct { pg: i64, scale: f32, is_causal: bool, - window: i64, - use_sink: bool, }; -/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. -fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { +/// Decode attention in Furiosa's form: the kernel gathers whole pages, then +/// runs one contraction, a masked softmax and a second contraction. It reads +/// past each sequence's end, selecting zero for those unused cache entries. +/// The token metadata depends only on the attention parameters, which every +/// layer shares: one kernel computes it once per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, cfg: DecodeCfg) Tensor { const columns = @divExact(cfg.a, cfg.pg); const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ @@ -236,7 +147,7 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), - }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal } }); return Decode.call(.{ .q = q.convert(.bf16), .pages = meta.pages, @@ -245,15 +156,19 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .keyed = meta.keyed, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = sink, }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; } -const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool }; -/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC -/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, -/// repro_row_broadcast_past_4096.py. Remove once both compile. +/// For each token: the cache page of each column, how many columns hold keys +/// it sees, the keys it sees (below its limit, in pages inside the cache) and +/// whether it sees any. Columns past the sequence's +/// pages read page `p`, so that the masked padding spreads over many pages +/// instead of hammering one. Rows are selected by one-hot products and +/// comparisons follow `attentionMask`'s clamp arithmetic: TCC 2026.5 has no +/// tactics for small integer gathers and fails on the plugin's lowering of the +/// same comparisons for several tokens. const DecodeMeta = tcl.Kernel(MetaCfg, .{ .name = "paged_decode_metadata", .inputs = &.{ "seq_lens", "query_offsets", "table" }, @@ -273,12 +188,14 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ }); const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + // Padding tokens follow every sequence's end and belong to none. const seq = try countBefore(b, ends, tokens, S); const own = try selector(b, seq, S, cfg.s); var op = b.tensorOperation(.{}); const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); var visible = try selectMetadata(b, t.seq_lens, own, S); if (cfg.is_causal) { + // Causal tokens see their sequence up to their own position. const end = try selectMetadata(b, ends, own, S); op = b.tensorOperation(.{}); visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); @@ -296,6 +213,8 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); op = b.tensorOperation(.{}); const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); + // Tokens without keys have no defined attention: the attention + // kernel zeroes them. op = b.tensorOperation(.{}); const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); op = b.tensorOperation(.{}); @@ -303,19 +222,21 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ op = b.tensorOperation(.{}); const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); op = b.tensorOperation(.{ .tactic = .elementwise }); - var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); - if (cfg.window > 0) { - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - } + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); b.ret(&.{ read, bounds, mask, keyed }); } }.run, }); +/// Gathers the key and value pages a token sees, contracts with the query, +/// masks keys past the sequence, normalizes in one softmax times `keyed` (zero +/// for tokens without keys, whose softmax is uniform) and contracts with the +/// values. A token's pages are gathered up to its bound, and the operations +/// reading them skip the rest: the work follows the sequence's length, not the +/// table's. Fusible: it joins the layer around it, one launch per layer. const Decode = tcl.Kernel(DecodeCfg, .{ .name = "paged_decode_attention", - .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache" }, .outputs = &.{"out"}, .run = struct { fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { @@ -337,360 +258,26 @@ const Decode = tcl.Kernel(DecodeCfg, .{ .keyed = .{ .dtype = .f32, .axes = &.{T} }, .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, - .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, }); - const sink = if (cfg.use_sink) t.sink else null; - const groups = @divExact(cfg.t, cfg.group); - if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); - const NG = b.axis("NG", groups); - const TG = b.axis("TG", @divExact(cfg.t, groups)); - var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); - { - const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); - const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); - const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); - const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); - const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); - const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); - } - b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); - } - }.run, -}); - -const unmasked_group_tokens = 4; - -/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. -fn decodeGroups(tokens: i64, keys: i64) i64 { - var groups: i64 = 1; - while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { - groups += 1; - while (@rem(tokens, groups) != 0) groups += 1; - } - return groups; -} - -/// `keyed` scales the softmax: zero for tokens without keys. -fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { - const T = q.axes[0]; - const Hkv = q.axes[1]; - const G = q.axes[2]; - const D = q.axes[3]; - const AP = pages.axes[1]; - const P = k_cache.axes[0]; - const PG = k_cache.axes[1]; - const keys: []const tcl.Axis = &.{ AP, PG }; - const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); - var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); - var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); - if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { - k = try maskCacheValues(b, k, key_mask); - v = try maskCacheValues(b, v, key_mask); - } - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); - op = b.tensorOperation(.{}); - const p = if (sink) |logits| blk: { - const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); - op = b.tensorOperation(.{}); - const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); - op = b.tensorOperation(.{}); - const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); - op = b.tensorOperation(.{}); - const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); - op = b.tensorOperation(.{}); - break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - } else blk: { - const x = op.fetch(masked, .{}); - const e = x.subf(x.reduce(keys, .maxf)).exp(); - break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - }; - op = b.tensorOperation(.{}); - return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); -} - -const head_keys = 4096; -const tail_chunk_keys = 8192; - -const HybridCfg = struct { - t: i64, - s: i64, - /// Head pages, tail chunks and pages per tail chunk. - hp: i64, - nc: i64, - cp: i64, - pg: i64, - hkv: i64, - g: i64, - d: i64, - pages: i64, - scale: f32, - is_causal: bool, -}; - -fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { - const pg = k_cache.dim(.k_chunk); - const t = q.dim(.b); - const s = parameters.seq_lens.dim(.b); - const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; - const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; - const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; - const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; - const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); - const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ - .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), - .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), - .hbounds = zml.Shape.init(.{ .t = t }, .i32), - .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), - .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), - .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), - .limits = zml.Shape.init(.{ .t = t }, .i32), - .keyed = zml.Shape.init(.{ .t = t }, .f32), - }, .{ .cfg = cfg }); - return HybridDecode.call(.{ - .q = q.convert(.bf16), - .hpages = meta.hpages, - .hmask = meta.hmask, - .hbounds = meta.hbounds, - .tpages = meta.tpages, - .tmask = meta.tmask, - .tbounds = meta.tbounds, - .limits = meta.limits, - .keyed = meta.keyed, - .k_cache = k_cache.convert(.bf16), - .v_cache = v_cache.convert(.bf16), - }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; -} - -/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the -/// longest token; `limits` is the visible key count. -const HybridMeta = tcl.Kernel(HybridCfg, .{ - .name = "paged_hybrid_metadata", - .inputs = &.{ "seq_lens", "query_offsets", "table" }, - .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, - .run = struct { - fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { - const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; - const T = b.axis("T", cfg.t); - const S = b.axis("S", cfg.s); - const O = b.axis("O", cfg.s + 1); - const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); - const HP = b.axis("HP", cfg.hp); - const RP = b.axis("RP", cfg.nc * cfg.cp); - const NC = b.axis("NC", cfg.nc); - const CP = b.axis("CP", cfg.cp); - const HA = b.axis("HA", cfg.hp * cfg.pg); - const CK = b.axis("CK", cfg.cp * cfg.pg); - const t = try b.declareArgs(.{ - .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, - .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, - .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, - }); - const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); - const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - const seq = try countBefore(b, ends, tokens, S); - const own = try selector(b, seq, S, cfg.s); + const mask = try b.reshape(t.mask, &.{ T, AP, PG }, .{}); + const k = try maskCacheValues(b, try b.gather(t.k_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); + const v = try maskCacheValues(b, try b.gather(t.v_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); + const keys: []const tcl.Axis = &.{ AP, PG }; var op = b.tensorOperation(.{}); - const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); - var visible = try selectMetadata(b, t.seq_lens, own, S); - if (cfg.is_causal) { - const end = try selectMetadata(b, ends, own, S); - op = b.tensorOperation(.{}); - visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); - } + const scores = try op.commit(op.contract(t.q, k, &.{ Hkv, G, T, AP, PG }).mulf(cfg.scale), .{}); op = b.tensorOperation(.{}); - const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); - - // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read - // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. - // Remove once it is exact on RNGD. - const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); op = b.tensorOperation(.{}); - const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); - // Trips never exceed nc: `limit` is at most the table's keys. - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); - var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); - { - const reads, const masks, const counts, const seen = loop.carried; - const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); - const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); - op = b.tensorOperation(.{}); - const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); - try loop.yield(.{ - try b.indexWrite(reads, loop.iv, chunk.pages), - try b.indexWrite(masks, loop.iv, chunk.mask), - try b.indexWrite(counts, loop.iv, chunk.bounds), - any, - }); - } - const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + const p = try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(t.keyed, .{})), .{ .dtype = .bf16 }); op = b.tensorOperation(.{}); - const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); - b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + b.ret(&.{try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 })}); } }.run, }); -/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. -fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { - const T = own.axes[0]; - const S = own.axes[1]; - const PG = b.axis("PG", cfg.pg); - const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); - var op = b.tensorOperation(.{}); - const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); - op = b.tensorOperation(.{}); - const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); - op = b.tensorOperation(.{}); - const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); - op = b.tensorOperation(.{}); - const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); - const to: []const tcl.Axis = &.{ T, cols, PG }; - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); - op = b.tensorOperation(.{}); - const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); - return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; -} - -const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; - -const HybridDecode = tcl.Kernel(HybridCfg, .{ - .name = "paged_hybrid_decode_attention", - .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { - b.fuse(); - const a = hybridAxes(b, cfg); - const t = try b.declareArgs(.{ - .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, - .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, - .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, - .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, - .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, - .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, - .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, - .limits = .{ .dtype = .i32, .axes = &.{a.T} }, - .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, - .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - }); - const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); - b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); - } - }.run, -}); - -const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; - -fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { - return .{ - .T = b.axis("T", cfg.t), - .HP = b.axis("HP", cfg.hp), - .HA = b.axis("HA", cfg.hp * cfg.pg), - .NC = b.axis("NC", cfg.nc), - .CP = b.axis("CP", cfg.cp), - .CK = b.axis("CK", cfg.cp * cfg.pg), - .Hkv = b.axis("Hkv", cfg.hkv), - .G = b.axis("G", cfg.g), - .D = b.axis("D", cfg.d), - .P = b.axis("P", cfg.pages), - .PG = b.axis("PG", cfg.pg), - }; -} - -fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { - const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); - var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); - { - const max, const sum, const out = loop.carried; - const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); - const bounds = try b.indexRead(tbounds, loop.iv); - const pages = try b.indexRead(tpages, loop.iv); - const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); - const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); - const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); - try loop.yield(.{ merged.max, merged.sum, merged.out }); - } - return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; -} - -fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { - const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); - const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); - const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); - const keys: []const tcl.Axis = &.{ a.HP, a.PG }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const sv = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); - op = b.tensorOperation(.{}); - const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); - const alpha = try expDifference(b, tail.max, max); - op = b.tensorOperation(.{}); - const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); - op = b.tensorOperation(.{}); - const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); - op = b.tensorOperation(.{}); - const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); - op = b.tensorOperation(.{}); - return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); -} - -fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { - const T = q.axes[0]; - const Hkv = q.axes[1]; - const G = q.axes[2]; - const D = q.axes[3]; - const keys = mask.axes[1..]; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const sv = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); - op = b.tensorOperation(.{}); - const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); - const exps = try maskedExp(b, masked, max, mask); - op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); - op = b.tensorOperation(.{}); - const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); - return .{ .max = max, .sum = sum, .out = out }; -} - -fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); - const alpha = try expDifference(b, x.max, max); - const beta = try expDifference(b, y.max, max); - op = b.tensorOperation(.{}); - const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); - op = b.tensorOperation(.{}); - const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); - return .{ .max = max, .sum = sum, .out = out }; -} - const Cfg = struct { tokens: i64, sequences: i64, @@ -712,9 +299,11 @@ const Cfg = struct { window: i32 = -1, }; -/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. +/// Attention tuning inherited from the original TCL backend, informed by +/// `furiosa.kernels.llama.config` and `common.attention`. Keep the compiler's +/// default lowering mode and omit the adaptive-einsum and half-VRF settings +/// removed in TCC 2026.5. fn configure(b: *tcl.Builder, cfg: Cfg) void { - const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; if (cfg.decode) b.compilerConfig(.{ .enable_einsum_fusion = true, .padding_policy = .{ .Small = 1.1 }, @@ -737,10 +326,11 @@ fn configure(b: *tcl.Builder, cfg: Cfg) void { .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, .tensor_unit_bridge_threshold_in_page = 12, .allow_external_operators = false, + .enable_tactic_pruning = false, .scheduler_beam_search = true, .allow_reduce_by_ve_cluster_chip_reduce = false, .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, .dma_preference = 1.2, }); } @@ -800,24 +390,11 @@ fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { const Prepared = struct { q: builder.Tensor, - /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). pages: builder.Tensor, - pick: ?builder.Tensor, qpos: builder.Tensor, kvlen: builder.Tensor, row: builder.Tensor, active: builder.Tensor, - /// Per page row: the first query position and the keys. - first: builder.Tensor, - keys: builder.Tensor, - - /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash - /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such - /// products compile. - fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { - const pick = self.pick orelse return pages; - return selectMetadata(b, pages, pick, a.S); - } }; fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { @@ -837,16 +414,15 @@ fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64 return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); } -// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for -// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, -// repro_i32_slice_odd_offset.py. Remove once both compile. +// Keep metadata exact in I32. Scalar gathers and four-byte-offset slices do +// not have native tactics for these shapes in TCC 2026.5. fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); } -/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would -/// need alignment staging at every invocation. +/// Keep adjacent I32 reads inside the native unit. Exposing offsets[1:] as a +/// separate argument would require alignment staging at every invocation. fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); @@ -856,7 +432,8 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); const seq_of_token = try countBefore(b, ends, tokens, a.S); - // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. + // All sequence ends precede a padding token. Reuse this vector instead of + // bringing a symbolic scalar reduction back into a vector operation. op = b.tensorOperation(.{}); const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); op = b.tensorOperation(.{}); @@ -876,19 +453,16 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis return .{ .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), - .pick = null, .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), .kvlen = try b.reshape(lengths, &.{a.B}, .{}), .row = token_seq, .active = active, - .first = try b.reshape(qpos, &.{a.B}, .{}), - .keys = try b.reshape(lengths, &.{a.B}, .{}), }; } - const shift = std.math.log2_int(u64, @intCast(cfg.i)); + const shift = std.math.log2_int(u64, prefill_block); op = b.tensorOperation(.{}); - const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(prefill_block - 1).binary(.shr_arith, shift), .{}); op = b.tensorOperation(.{}); const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); op = b.tensorOperation(.{}); @@ -908,7 +482,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); op = b.tensorOperation(.{}); - const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(prefill_block), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); op = b.tensorOperation(.{}); @@ -926,15 +500,14 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); op = b.tensorOperation(.{}); const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); + const table = try selectMetadata(b, t.pages, pick, a.S); const token_pick = try selector(b, token_seq, a.S, cfg.sequences); const token_begin = try selectMetadata(b, starts, token_pick, a.S); const token_block = try selectMetadata(b, block_start, token_pick, a.S); op = b.tensorOperation(.{}); - const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); - op = b.tensorOperation(.{}); - const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); - return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; + const row = try op.commit(op.fetch(token_block, .{}).muli(prefill_block).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + return .{ .q = q, .pages = table, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active }; } fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { @@ -960,7 +533,8 @@ fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.Finish return .{ .safe = safe, .valid = valid }; } -/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. +/// Cache validity is per sequence, not per query: causal/window masking stays +/// on scores, without replicating K/V for every row of a prefill block. fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { var op = b.tensorOperation(.{}); const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); @@ -968,15 +542,17 @@ fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); } -/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. +/// A true select excludes NaN/Inf in unwritten slots; multiplying by zero +/// would still propagate them into the contraction. Materialize only BF16. fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = .elementwise }); const data = op.fetch(values, .{}); return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); } +/// Build the sequence, query, page and window mask for the current keys. fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { - const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[0] }; const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; // Elementwise operations require their first read to cover the output // axes, and each VE instruction must consume the current accumulator. @@ -1001,6 +577,8 @@ fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder return mask; } +/// Keep the contraction accumulator in f32 through masking and softmax. +/// A finite sentinel also makes entirely masked chunks safe to rescale. fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; var op = b.tensorOperation(.{}); @@ -1013,7 +591,8 @@ fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builde fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{}); const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); - // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. + // Subtracting the sentinel from itself gives exp(0), so masking only + // the logits is insufficient for empty rows and fully masked chunks. return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); } @@ -1043,7 +622,7 @@ const OneShot = tcl.Kernel(Cfg, .{ const prepared = try prepareQueries(b, cfg, a, t); const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); + const page_indices = try validIndices(b, prepared.pages, cfg.pages); const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); @@ -1051,6 +630,7 @@ const OneShot = tcl.Kernel(Cfg, .{ slot_op = b.tensorOperation(.{}); const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); + // Match the original slot sanitization before reading invalid pages. slot_op = b.tensorOperation(.{}); const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); const valid_length = if (cfg.decode) prepared.kvlen else null; @@ -1062,6 +642,8 @@ const OneShot = tcl.Kernel(Cfg, .{ const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); var op = b.tensorOperation(.{}); const p = if (cfg.decode) compact: { + // Furiosa's common.softmax keeps normalization inside one VE + // operation. FP32 intermediates only narrow at the DPE input. const exp_sink = if (cfg.use_sink) sink: { const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); break :sink try expDifference(b, t.sink, maximum); @@ -1074,11 +656,13 @@ const OneShot = tcl.Kernel(Cfg, .{ var sum = e.reduce(&.{A}, .addf); if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); - // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. + // Empty rows have uniform probabilities from finite sentinels. + // Clear masked probabilities before contracting with values. op = b.tensorOperation(.{}); break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); } else tiled: { - // Separate reductions schedule better for multi-query blocks with the current compiler. + // Separate reductions schedule better for multi-query blocks + // with the current compiler; keep their intermediate state FP32. var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); const maximum = try op.commit(m, .{}); @@ -1111,66 +695,28 @@ const Flash = tcl.Kernel(Cfg, .{ const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); const t = try declareInputs(b, a); const prepared = try prepareQueries(b, cfg, a, t); - const rows = prepared.pages.axes[0]; - const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); + const by_batch = try b.reshape(prepared.pages, &.{ a.B, C, Cp }, .{}); const page_op = b.tensorOperation(.{}); - const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); + const pages = try page_op.commit(page_op.fetch(by_batch, .{}), .{ .axes = &.{ C, a.B, Cp } }); const stats = &.{ a.Hkv, a.G, a.B, a.I }; const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; var op = b.tensorOperation(.{}); - const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); - // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides - // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. - const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { - op = b.tensorOperation(.{}); - const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); - op = b.tensorOperation(.{}); - const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); - // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per - // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index - // compiles as data. - const trip = try b.arange(C.size, .i32, &.{C}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); - break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; - } else null; - const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); { const sum_acc, const qkv_acc, const max_acc = loop.carried; - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - var row_pages: builder.Tensor = undefined; - var position: builder.Tensor = undefined; - var first_keys: builder.Tensor = undefined; - if (window) |w| { - const chunk_index = try b.indexRead(w.chunks, loop.iv); - op = b.tensorOperation(.{}); - const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); - // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower - // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove - // once batched gathers of several rows compile. - row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); - op = b.tensorOperation(.{}); - const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); - const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; - op = b.tensorOperation(.{}); - position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); - op = b.tensorOperation(.{}); - first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); - } else { - row_pages = try b.indexRead(pages, loop.iv); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - op = b.tensorOperation(.{}); - position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - op = b.tensorOperation(.{}); - first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); - } - const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); + const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); const idx = indices.safe; op = b.tensorOperation(.{}); const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + op = b.tensorOperation(.{}); + const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); op = b.tensorOperation(.{}); @@ -1178,8 +724,9 @@ const Flash = tcl.Kernel(Cfg, .{ op = b.tensorOperation(.{}); const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); - const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); op = b.tensorOperation(.{}); const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); @@ -1204,6 +751,7 @@ const Flash = tcl.Kernel(Cfg, .{ var sum = loop.results[0]; var numerator = loop.results[1]; if (cfg.use_sink) { + // Merge the zero-valued sink once, after all key chunks. op = b.tensorOperation(.{}); const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); const alpha = try expDifference(b, loop.results[2], maximum); @@ -1374,7 +922,6 @@ const TestDims = struct { page: usize = 16, uniform: bool = false, uniform_scores: bool = false, - spikes: []const usize = &.{}, dtype: zml.DataType = .bf16, }; @@ -1483,7 +1030,7 @@ test "tcl paged attention cache masking emits a BF16 select without query replic try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.ve_opcode<\"mulf\">") == null); try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); - const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true }); defer std.testing.allocator.free(decode); try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); @@ -1596,32 +1143,6 @@ test "tcl paged attention chunked decode on furiosa" { try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); } -test "tcl paged attention hybrid decode on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(2049, 2101, &.{ - .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, - .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, - }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); - try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); -} - -test "tcl paged attention flash decode of many tokens on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); - try runCases(2049, 2600, &.{ - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, - }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); - try runCases(2049, 2700, &.{ - .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, - .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, - }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); -} - test "tcl paged attention uniform probabilities on furiosa" { if (zml.testing.env().target != .furiosa) return error.SkipZigTest; try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); @@ -1651,11 +1172,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { - return paged.pagedAttention(p, q, k, v, opts); + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { + return paged_attention.pagedAttention(.{ .tcl = p }, q, k, v, .{ .split = .{ .k = k, .v = v } }, opts); } }; - const opts: AttentionOptions = .{ + const opts: paged_attention.AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1688,16 +1209,9 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); - if (dims.spikes.len > 0) { - for (case.seqs[0..batch], 0..) |seq, row| { - for (0..@intCast(seq[1])) |key| { - const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); - const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; - @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); - } - } - } if (case.poison != .none) { + // Mark the union of live physical slots before poisoning: reordered + // or shared pages must retain every token a live sequence can read. const live = try allocator.alloc(bool, num_pages * page); defer allocator.free(live); @memset(live, false); @@ -1729,11 +1243,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(AttentionOptions) = .{ + var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 2a12e012..1ccbeba8 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,10 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../options.zig").AttentionOptions; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_oneapi = @import("unified_attention_oneapi.zig"); const kernels_sparse = @import("sparse_unified_attention.zig"); +const kernels_oneapi = @import("unified_attention_oneapi.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -274,57 +274,92 @@ pub const paged = struct { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); } - const cu_count = getCuCount(); - const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); - const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); - const num_queries_per_kv: usize = num_heads / num_kv_heads; - // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the - // single decode query token doesn't carry masked-out fp32 acc lanes. - // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) - std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) - else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(q.dim(.b)); - const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); - const total_q_blocks: usize = num_tokens / block_q + num_seqs; - const target_num_prgms: usize = cu_count * 4; - const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + const output = zml.ops.manualComputation( + (struct { + q: zml.Tensor, + k_cache: zml.Tensor, + v_cache: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: AttentionOptions, + options: Options, - const sliding_window: usize = opts.mask.slidingWindow() orelse 0; + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const parameters_: Parameters = .{ + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, + }; - const paged_attention_opts: PagedAttentionOptions = .{ - .cu_count = getCuCount(), - .all_decode = !parameters.options_.is_prefill, - .num_tokens = num_tokens, - .num_heads = num_heads, - .num_kv_heads = num_kv_heads, - .head_dim = @intCast(q.dim(.hd)), - .batch_size = @intCast(parameters.block_table.dim(.b)), - .block_size = @intCast(k_cache.dim(.k_chunk)), - .num_blocks = @intCast(k_cache.dim(.page)), - .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), - .sliding_window = sliding_window, - .block_m = block_m, - .block_q = block_q, - .total_q_blocks = total_q_blocks, - .target_num_prgms = target_num_prgms, - .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = parameters.options_.max_seqlen_q, - .scale = opts.scale, - }; + const cu_count = getCuCount(); + const num_heads: usize = @intCast(self.q.dim(.hkv) * self.q.dim(.hg)); + const num_kv_heads: usize = @intCast(self.k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(self.q.dim(.b)); + const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - const use_2d_kernel = use2dKernel( - paged_attention_opts.all_decode, - paged_attention_opts.batch_size, - paged_attention_opts.num_kv_heads, + const sliding_window: usize = self.opts.mask.slidingWindow() orelse 0; + + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !self.options.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(self.q.dim(.hd)), + .batch_size = @intCast(parameters_.block_table.dim(.b)), + .block_size = @intCast(self.k_cache.dim(.k_chunk)), + .num_blocks = @intCast(self.k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = self.options.max_seqlen_q, + .scale = self.opts.scale, + }; + + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, + ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) + else + pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); + + return output; + } + }).body, + .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + q.shape(), ); - const output = if (use_2d_kernel) - pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) - else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) - else - pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); return output; } @@ -337,18 +372,51 @@ pub const paged = struct { stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); - const config = selectSparseConfig( - @intCast(opts.mask.indices.dim(.topk)), - @intCast(parameters.block_table.dim(.b)), - @intCast(q.dim(.hkv)), - getCuCount(), - !parameters.options_.is_prefill, - ); + return zml.ops.manualComputation( + (struct { + q: zml.Tensor, + k_cache: zml.Tensor, + v_cache: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: AttentionOptions, + options: Options, - return if (config.num_segments == 1) - pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) - else - pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const parameters_: Parameters = .{ + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, + }; + + const config = selectSparseConfig( + @intCast(self.opts.mask.indices.dim(.topk)), + @intCast(self.block_table.dim(.b)), + @intCast(self.q.dim(.hkv)), + getCuCount(), + !self.options.is_prefill, + ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config) + else + pagedSparseAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config); + } + }).body, + .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + q.shape(), + ); } pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index 7b2b9428..39018f1b 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -1020,25 +1020,6 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - const output_offset = query_token_idx.to(.i64).mul(output_stride_0) - .add(query_head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); - - // The launch covers the padded query capacity, but the attention pass only - // writes scratch for active tokens. Zero the tail before looking up a - // sequence or dividing by its segment size (zero for inactive sequences). - const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); - const inactive = query_token_idx.ge(active_tokens); - k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ - .mask = dim_mask.bitAnd(inactive), - }); - k.returnIf(inactive); - const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); @@ -1048,6 +1029,12 @@ fn reduceSegments( const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); @@ -1113,6 +1100,9 @@ fn reduceSegments( ); } + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); k.storeOpts( output_ptr.addPtr(output_offset), acc.to(config.o_dtype), diff --git a/zml/mem.zig b/zml/mem.zig index a64ae968..596ebe85 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -1,320 +1,9 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; -const Alignment = std.mem.Alignment; -const assert = std.debug.assert; - -const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; -const Device = @import("platform.zig").Device; -const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); -const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; -const log = std.log.scoped(.@"zml/mem"); - -pub const DmaAllocator = union(enum) { - passthrough: std.mem.Allocator, - uib: UninitializedBufferAllocator, - dmam: DmaMapAllocator, - - pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { - return switch (device.platform.target) { - .cuda => .{ .dmam = .init(parent, device.platform) }, - .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal => .{ .passthrough = parent }, - }; - } - - pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { - return switch (self.*) { - .passthrough => |a| a, - inline else => |*a| a.allocator(), - }; - } -}; - -pub const UninitializedBufferAllocator = struct { - memory: *const Memory, - - const Header = struct { - buffer: *pjrt.Buffer, - }; - - pub fn init(memory: *const Memory) UninitializedBufferAllocator { - return .{ - .memory = memory, - }; - } - - pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const pjrt_client = self.memory.platform.pjrt_client; - - const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); - - const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ - .dims = &.{@intCast(total_len)}, - .element_type = .u8, - .layout = .{ - .tiled = .{ - .minor_to_major = &.{0}, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .dst = .{ .memory = self.memory.pjrt_memory }, - }) catch return null; - - const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); - const data_with_header: []u8 = opaque_ptr[0..total_len]; - - const header = std.mem.bytesAsValue(Header, opaque_ptr); - header.* = .{ - .buffer = pjrt_buffer, - }; - const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); - return @ptrCast(data_with_header[offset..]); - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { - _ = ret_addr; - const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); - const pjrt_api = self.memory.platform.pjrt_api; - const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); - header.buffer.deinit(pjrt_api); - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } -}; - -pub const DmaMapAllocator = struct { - parent: std.mem.Allocator, - platform: *const Platform, - - pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { - return .{ - .parent = parent, - .platform = platform, - }; - } - - pub fn allocator(self: *const DmaMapAllocator) Allocator { - return .{ - .ptr = @constCast(self), - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }, - }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - const allocation = self.parent.rawAlloc(len, alignment, ret_addr); - if (allocation) |loc| { - const data = loc[0..len]; - self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { - self.parent.rawFree(data, alignment, ret_addr); - return null; - }; - } - return allocation; - } - - fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return false; - } - - fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - _ = ctx; - _ = buf; - _ = alignment; - _ = new_len; - _ = ret_addr; - return null; - } - - fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { - const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); - self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; - self.parent.rawFree(buf, alignment, ret_addr); - } -}; - -pub const FixedBufferPool = struct { - buffer: []u8, - block_size: usize, - q_buf: []const u16, - q: std.Io.Queue(u16), - - pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { - const block_size = @divExact(buffer_.len, blocks_); - const q_buf = try allocator.alloc(u16, blocks_); - for (q_buf, 0..) |*idx, i| { - idx.* = @intCast(i); - } - var q: std.Io.Queue(u16) = .init(q_buf); - q.type_erased.len = q.type_erased.buffer.len; // make the queue full - return .{ - .buffer = buffer_, - .block_size = block_size, - .q = q, - .q_buf = q_buf, - }; - } - - pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { - allocator.free(self.q_buf); - } - - pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { - const idx = try self.q.getOneUncancelable(io); - return self.buffer[idx * self.block_size ..][0..self.block_size]; - } - - fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { - return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and - @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); - } - - pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { - // is the pointer in range ? - std.debug.assert(inRange(buf, self.buffer)); - const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); - self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; - } -}; - -pub const DynamicBufferPool = struct { - const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .of(Node); - - block_size: usize, - max_blocks: usize, - - arena_state: std.heap.ArenaAllocator.State = .{}, - free_stack: std.atomic.Value(?*Node) = .init(null), - in_flight: std.atomic.Value(usize) = .init(0), - mutex: std.Io.Mutex = .init, - condition: std.Io.Condition = .init, - - pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { - return .{ - .block_size = block_size, - .max_blocks = max_blocks, - }; - } - - pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { - self.arena_state.promote(allocator).deinit(); - } - - pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { - while (true) { - var in_flight = self.in_flight.load(.acquire); - while (in_flight < self.max_blocks) { - if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { - in_flight = actual; - continue; - } - - if (self.pop()) |node| { - return self.nodeToSlice(node); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - var arena = self.arena_state.promote(allocator); - defer self.arena_state = arena.state; - - errdefer _ = self.in_flight.fetchSub(1, .release); - const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); - return @alignCast(buffer); - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (self.in_flight.load(.acquire) >= self.max_blocks) { - try self.condition.wait(io, &self.mutex); - } - } - } - - pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { - const node: *Node = @ptrCast(@alignCast(buf.ptr)); - var head = self.free_stack.load(.acquire); - while (true) { - node.next = head; - if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { - head = actual_head; - } else { - break; - } - } - - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - _ = self.in_flight.fetchSub(1, .release); - self.condition.signal(io); - } - - fn pop(self: *DynamicBufferPool) ?*Node { - var head = self.free_stack.load(.acquire); - while (true) { - if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { - head = actual; - continue; - } - - return head; - } - } - - fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { - const ptr: [*]u8 = @ptrCast(node); - return ptr[0..self.block_size]; - } -}; - /// Return a clone of a type with Tensors replaced by Buffer. /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. @@ -323,6 +12,21 @@ pub fn Bufferized(comptime T: type) type { return meta.MapRestrict(Tensor, Buffer).map(T); } +/// The field of `Bufferized(Model)` that holds the struct `Model`'s field +/// `name`, or null when `Bufferized` strips it. `Bufferized` keeps the +/// fields holding tensors, in order, and renumbers a tuple's. +fn bufferizedField(comptime Model: type, comptime name: []const u8) ?[]const u8 { + @setEvalBranchQuota(10_000); + const info = @typeInfo(Model).@"struct"; + var index: usize = 0; + for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.@"comptime" or !meta.Contains(Field, Tensor)) continue; + if (std.mem.eql(u8, field_name, name)) return @typeInfo(Bufferized(Model)).@"struct".field_names[index]; + index += 1; + } + return null; +} + fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { @setEvalBranchQuota(10_000); const Model = @TypeOf(model); @@ -341,8 +45,16 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.field_names) |field_name| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + var initialized_fields: usize = 0; + errdefer inline for (struct_type_info.field_names, 0..) |field_name, index| { + if (index < initialized_fields) + deinitBufferized(allocator, &@field(bufferized_, field_name)); + }; + inline for (@typeInfo(Model).@"struct".field_names) |field_name| { + if (comptime bufferizedField(Model, field_name)) |buffer_field| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, buffer_field)); + initialized_fields += 1; + } } }, .@"union" => { @@ -364,23 +76,333 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); - for (model, bufferized_.*) |src, *dst| { - try bufferizeInner(allocator, src, dst); + const allocated = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + var initialized: usize = 0; + errdefer { + for (allocated[0..initialized]) |*element| deinitBufferized(allocator, element); + allocator.free(allocated); } + for (model, allocated) |src, *dst| { + try bufferizeInner(allocator, src, dst); + initialized += 1; + } + bufferized_.* = allocated; }, else => unreachable, } }, + .array => |info| { + var initialized: usize = 0; + errdefer for (bufferized_.*[0..initialized]) |*element| deinitBufferized(allocator, element); + inline for (0..info.len) |index| { + try bufferizeInner(allocator, model[index], &bufferized_.*[index]); + initialized = index + 1; + } + }, .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, else => unreachable, } } -/// Convert a model to its bufferized form by replacing Tensor fields with Buffer -/// and allocating any required slices using the provided allocator. -pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { - var bufferized: Bufferized(ModelType) = undefined; +/// Convert the model `model` points to into its bufferized form by replacing +/// Tensor fields with Buffer and allocating any required slices using the +/// provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model.*)) { + var bufferized: Bufferized(@TypeOf(model.*)) = undefined; try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } + +/// A tensor of a model, the buffer at its place in the model's buffers, and +/// its field path from the root, as `.layers[1].ffn.weight`. +pub const Pair = struct { tensor: *const Tensor, buffer: *Buffer, path: []const u8 }; + +/// Pairs each tensor of the model `model` points to with the buffer at its +/// place in `bufferized`, in the order `meta.visit` finds the tensors. The +/// pairs and their paths live in `arena`. The two trees must have the +/// structure `bufferize` gives: the same union arms, the same optionals set, +/// slices of the same lengths. `report`, an empty fixed writer, builds the +/// paths and cuts them short when full. At the first place the trees differ +/// the walk stops and returns `error.Mismatch`, and `report` ends with the +/// path there and what differs. +pub fn pairBufferized( + arena: std.mem.Allocator, + model: anytype, + bufferized: *Bufferized(@TypeOf(model.*)), + report: *std.Io.Writer, +) error{ Mismatch, OutOfMemory }![]Pair { + var pairs: std.ArrayList(Pair) = try .initCapacity(arena, meta.count(Tensor, model)); + try pairBufferizedInner(arena, model, bufferized, report, &pairs); + return pairs.items; +} + +/// `report.end` is restored after each part that pairs, so at a tensor or a +/// mismatch the report holds the path down to it. +fn pairBufferizedInner( + arena: std.mem.Allocator, + model: anytype, + bufferized: anytype, + report: *std.Io.Writer, + pairs: *std.ArrayList(Pair), +) error{ Mismatch, OutOfMemory }!void { + const Model = @typeInfo(@TypeOf(model)).pointer.child; + if (comptime !meta.Contains(Model, Tensor)) return; + if (Model == Tensor) return pairs.appendAssumeCapacity(.{ + .tensor = model, + // `bufferize` allocates every slice mutable, a `[]const` one included. + .buffer = @constCast(bufferized), + .path = try arena.dupe(u8, report.buffered()), + }); + + const depth = report.end; + switch (@typeInfo(Model)) { + .@"struct" => |info| inline for (info.field_names) |name| { + if (comptime bufferizedField(Model, name)) |buffer_field| { + report.print(".{s}", .{name}) catch {}; + try pairBufferizedInner(arena, &@field(model, name), &@field(bufferized, buffer_field), report, pairs); + report.end = depth; + } + }, + .@"union" => { + const tag = std.meta.activeTag(model.*); + if (tag != std.meta.activeTag(bufferized.*)) { + report.print(": union arm .{t} in the model, .{t} in the buffers", .{ tag, std.meta.activeTag(bufferized.*) }) catch {}; + return error.Mismatch; + } + switch (model.*) { + inline else => |*payload, arm| { + report.print(".{s}", .{@tagName(arm)}) catch {}; + try pairBufferizedInner(arena, payload, &@field(bufferized, @tagName(arm)), report, pairs); + report.end = depth; + }, + } + }, + .optional => { + if ((model.* == null) != (bufferized.* == null)) { + report.print(": {s} in the model, {s} in the buffers", .{ if (model.* == null) "null" else "set", if (bufferized.* == null) "null" else "set" }) catch {}; + return error.Mismatch; + } + if (model.*) |*payload| try pairBufferizedInner(arena, payload, &bufferized.*.?, report, pairs); + }, + .pointer => |info| { + if (info.size != .slice) @compileError("mem.pairBufferized: " ++ @typeName(Model) ++ " is not a slice"); + if (model.len != bufferized.len) { + report.print(": a slice of {d} in the model, of {d} in the buffers", .{ model.len, bufferized.len }) catch {}; + return error.Mismatch; + } + for (model.*, bufferized.*, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + } + }, + .array => for (model, bufferized, 0..) |*element, *buffer, i| { + report.print("[{d}]", .{i}) catch {}; + try pairBufferizedInner(arena, element, buffer, report, pairs); + report.end = depth; + }, + else => @compileError("mem.pairBufferized: cannot walk " ++ @typeName(Model)), + } +} + +/// Deinitializes every accelerator buffer of the tree `value` points to and +/// frees the recursive slice storage allocated by `bufferize`. +pub fn deinitBufferized(allocator: std.mem.Allocator, value: anytype) void { + const Ptr = @TypeOf(value); + const T = @typeInfo(Ptr).pointer.child; + if (T == Tensor) @compileError("mem.deinitBufferized takes the buffers bufferize returns, not the model"); + if (T == Buffer) { + const buffer: *Buffer = @constCast(value); + buffer.deinit(); + return; + } + switch (@typeInfo(T)) { + .@"struct" => |info| inline for (info.field_names) |field_name| { + deinitBufferized(allocator, &@field(value, field_name)); + }, + .@"union" => switch (value.*) { + inline else => |*payload| deinitBufferized(allocator, payload), + }, + .optional => if (value.*) |*payload| { + deinitBufferized(allocator, payload); + }, + .pointer => |info| switch (info.size) { + .slice => { + for (value.*) |*element| deinitBufferized(allocator, element); + allocator.free(value.*); + }, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + }, + .array => for (&value.*) |*element| deinitBufferized(allocator, element), + // What `Bufferized` makes of a union arm or a whole tree without a tensor. + .void => {}, + else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), + } +} + +test "bufferize maps a tuple's fields as Bufferized renumbers them" { + const Model = struct { u32, Tensor, []Tensor, [2]Tensor }; + const a: Tensor = .init(.{2}, .f32); + const b: Tensor = .init(.{3}, .f16); + var tensors = [_]Tensor{ a, b }; + const model: Model = .{ 3, a, &tensors, .{ b, a } }; + var bufferized = try bufferize(std.testing.allocator, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); + try std.testing.expect(bufferized[0].shape().eql(a.shape())); + try std.testing.expectEqual(2, bufferized[1].len); + try std.testing.expect(bufferized[1][1].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][0].shape().eql(b.shape())); + try std.testing.expect(bufferized[2][1].shape().eql(a.shape())); +} + +test "bufferize rolls back earlier slice fields on allocation failure" { + const Model = struct { + first: []const Tensor, + second: []const Tensor, + }; + var tensors: [1]Tensor = undefined; + const model: Model = .{ .first = &tensors, .second = &tensors }; + var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 1 }); + + if (bufferize(failing.allocator(), &model)) |result| { + var unexpected = result; + deinitBufferized(failing.allocator(), &unexpected); + return error.ExpectedOutOfMemory; + } else |err| { + try std.testing.expect(err == error.OutOfMemory); + } + try std.testing.expect(failing.has_induced_failure); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "deinitBufferized frees recursive const slices" { + const Layer = struct { weights: []const Tensor }; + const Model = struct { + layers: []const Layer, + fixed: [2]Tensor, + }; + var weights: [2]Tensor = undefined; + const layers = [_]Layer{ + .{ .weights = weights[0..1] }, + .{ .weights = weights[1..2] }, + }; + const model: Model = .{ .layers = &layers, .fixed = undefined }; + var bufferized = try bufferize(std.testing.allocator, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} + +test "deinitBufferized frees slices under optionals and unions" { + const Layer = struct { weights: []const Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []const Layer, none: void }; + const Model = struct { + present: ?Layer, + absent: ?Layer, + ffns: [3]Ffn, + }; + var weights: [3]Tensor = undefined; + const experts = [_]Layer{ .{ .weights = weights[0..1] }, .{ .weights = weights[1..3] } }; + const model: Model = .{ + .present = .{ .weights = &weights }, + .absent = null, + .ffns = .{ .{ .dense = .{ .weights = weights[2..3] } }, .{ .experts = &experts }, .none }, + }; + var bufferized = try bufferize(std.testing.allocator, &model); + deinitBufferized(std.testing.allocator, &bufferized); +} + +test "pairBufferized pairs tensors and buffers in visit order" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, experts: []Layer, none: void }; + const Model = struct { ffns: [3]Ffn, head: Tensor, layers: u32 }; + var weights: [4]Tensor = undefined; + var experts = [_]Layer{ .{ .weights = weights[1..2], .scale = null }, .{ .weights = weights[2..4], .scale = weights[3] } }; + var model: Model = .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = weights[0] } }, .{ .experts = &experts }, .none }, + .head = undefined, + .layers = 2, + }; + var bufferized = try bufferize(std.testing.allocator, &model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + var report_buffer: [64]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + const pairs = try pairBufferized(arena.allocator(), &model, &bufferized, &report); + try std.testing.expectEqual(0, report.end); + + const Visited = struct { + tensors: [8]*const Tensor = undefined, + buffers: [8]*Buffer = undefined, + tensor_count: usize = 0, + buffer_count: usize = 0, + }; + var visited: Visited = .{}; + meta.forEachVisit(&model, *const Tensor, struct { + fn call(_: usize, tensor: *const Tensor, out: *Visited) void { + out.tensors[out.tensor_count] = tensor; + out.tensor_count += 1; + } + }.call, .{&visited}); + meta.forEachVisit(&bufferized, *Buffer, struct { + fn call(_: usize, buffer: *Buffer, out: *Visited) void { + out.buffers[out.buffer_count] = buffer; + out.buffer_count += 1; + } + }.call, .{&visited}); + const paths = [_][]const u8{ + ".ffns[0].dense.weights[0]", + ".ffns[0].dense.scale", + ".ffns[1].experts[0].weights[0]", + ".ffns[1].experts[1].weights[0]", + ".ffns[1].experts[1].weights[1]", + ".ffns[1].experts[1].scale", + ".head", + }; + try std.testing.expectEqual(paths.len, pairs.len); + try std.testing.expectEqual(pairs.len, visited.tensor_count); + try std.testing.expectEqual(pairs.len, visited.buffer_count); + // Not `expectEqualSlices`: printing a `*Buffer` reaches code a CPU test build lacks. + for (pairs, paths, 0..) |pair, path, i| { + try std.testing.expect(pair.tensor == visited.tensors[i]); + try std.testing.expect(pair.buffer == visited.buffers[i]); + try std.testing.expectEqualStrings(path, pair.path); + } +} + +test "pairBufferized stops at the first place the trees differ and names it" { + const Layer = struct { weights: []Tensor, scale: ?Tensor }; + const Ffn = union(enum) { dense: Layer, none: void }; + const Model = struct { ffns: []Ffn, head: Tensor }; + var weights: [3]Tensor = undefined; + var built = [_]Ffn{ .{ .dense = .{ .weights = weights[0..2], .scale = null } }, .none }; + const built_model: Model = .{ .ffns = &built, .head = undefined }; + var bufferized = try bufferize(std.testing.allocator, &built_model); + defer deinitBufferized(std.testing.allocator, &bufferized); + + const Case = struct { ffns: [2]Ffn, report: []const u8 }; + const cases = [_]Case{ + .{ .ffns = .{ .none, .none }, .report = ".ffns[0]: union arm .none in the model, .dense in the buffers" }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..2], .scale = weights[2] } }, .none }, + .report = ".ffns[0].dense.scale: set in the model, null in the buffers", + }, + .{ + .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = null } }, .none }, + .report = ".ffns[0].dense.weights: a slice of 1 in the model, of 2 in the buffers", + }, + }; + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + for (cases) |case| { + var ffns = case.ffns; + const model: Model = .{ .ffns = &ffns, .head = undefined }; + var report_buffer: [128]u8 = undefined; + var report: std.Io.Writer = .fixed(&report_buffer); + // Not `expectError`: printing a `*Buffer` reaches code a CPU test build lacks. + if (pairBufferized(arena.allocator(), &model, &bufferized, &report)) |_| { + return error.TestExpectedMismatch; + } else |err| try std.testing.expectEqual(error.Mismatch, err); + try std.testing.expectEqualStrings(case.report, report.buffered()); + } +} diff --git a/zml/meta.zig b/zml/meta.zig index 84fb9d36..504b2973 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -143,7 +143,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. +/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -180,11 +180,6 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } - if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { - to.* = from; - return; - } - if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { @@ -231,11 +226,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => { - const item = try allocator.create(ptr_info.child); - try mapAlloc(cb, allocator, ctx, from.*, item); - to.* = item; - }, + else => try mapAlloc(cb, allocator, ctx, from.*, to.*), }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -276,7 +267,6 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, - pointer: *const A, other: u8, tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. @@ -288,7 +278,6 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, - pointer: *const B, other: u8, tag: [*:0]const u8, static_val: u8, @@ -302,7 +291,6 @@ test mapAlloc { .other = 7, .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, - .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -310,7 +298,6 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); - defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); @@ -322,7 +309,6 @@ test mapAlloc { try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); - try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: @@ -346,6 +332,13 @@ pub fn MapRestrict(From: type, To: type) type { if (!Contains(T, From)) return void; + // `visit` walks a bounded array's `len` elements, but its mapped + // struct would hold the whole buffer: pairing the two by visit + // order would miss or misplace elements. + if (@typeInfo(T) == .@"struct" and @hasDecl(T, "constSlice") and @hasDecl(T, "slice")) { + stdx.debug.compileError("zml.meta.MapRestrict({any}, {any}) cannot map the bounded array {any}: use a slice or an array", .{ From, To, T }); + } + return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. diff --git a/zml/nn.zig b/zml/nn.zig index eceb316b..e8d9873d 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,13 +2,11 @@ const std = @import("std"); const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -33,10 +31,6 @@ pub const Linear = struct { }; } - pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { - zml.Buffer.deinitAll(Linear, self); - } - pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { return if (self.quantization) |q| q.scheme else null; } @@ -228,8 +222,8 @@ test "block128 scaled dot layouts" { const io = std.testing.io; const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) - if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + if (platform.target != .cuda) return error.SkipZigTest; + const dtype = Quantization.Target.of(platform).fp8.?; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; @@ -322,23 +316,13 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - // Keep the unmatched-composite marker in the graph for backend lowering. - const ctx = zml.Compiler.current(); - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, - &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, - .{ - .call_target_name = "zml$scaled_dot_unmatched", - .has_side_effect = false, - .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); - } - return Tensor._result(out_shape, op.result(0)); + return ops.customCall( + "zml$scaled_dot_unmatched", + .{ in[0], in[1], in[2], in[3] }, + out_shape, + {}, + .{ .has_side_effect = false }, + ); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { @@ -2366,8 +2350,8 @@ pub fn softplus( // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. const z = x.exp(); const u = z.addConstant(1); - const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); - return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); + const log1p = u.cmp(.EQ, .scalar(1, .f32)).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, .f32)).select(x, log1p); } test "softplus" { @@ -2377,11 +2361,14 @@ test "softplus" { const platform = zml.testing.env(); const input: Tensor = .init(.{ .x = inputs.len }, .f32); const Local = struct { - fn call(x: Tensor, dtype: DataType) Tensor { - return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); + fn call(x: Tensor) Tensor { + return softplus(x, .{ .threshold = 2.0 }); } }; + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{input}, .{}); + defer exe.deinit(); + var input_buffer: zml.Buffer = try .fromBytes( std.testing.io, platform, @@ -2390,18 +2377,13 @@ test "softplus" { ); defer input_buffer.deinit(); - for ([_]DataType{ .f32, .bf16 }) |dtype| { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); - defer exe.deinit(); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer result.deinit(); - - try zml.testing.expectClose( - std.testing.io, - Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), - result, - .{}, - ); - } + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); } diff --git a/zml/ops.zig b/zml/ops.zig index f8fdba7e..b1a26d23 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -8,6 +8,13 @@ const platforms = @import("platforms"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; + +test { + _ = manual_computation; +} + const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -16,113 +23,15 @@ const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; -pub const ManualPartitioning = manual_computation.ManualPartitioning; -pub const PartitionSpecs = manual_computation.PartitionSpecs; -pub const partitionSpecs = manual_computation.partitionSpecs; -pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; - -test { - _ = manual_computation; -} - const log = std.log.scoped(.@"zml/Compiler"); -/// Reduces local tensor values across all processes in the computation. -/// For each tensor index, `func_` combines the values at that index from every -/// process. The result is identical on all processes. -/// -/// Inside a manual computation, `inputs` describes the local tensors at each -/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives -/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. -/// -/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. -/// Each input must have matching shapes and dtypes across processes. -/// * `func_` describes how to combine two scalar contributions: -/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such -/// as `Tensor.add`. This also applies to a one-element tuple or array. -/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a -/// tuple or array with one scalar tensor per input, in the same order. -/// -/// Returns the same container type as `inputs`, preserving tensor attributes. -/// With one process, returns `inputs` unchanged. -/// -/// The reduction uses one group containing all processes, including mesh replicas. -/// To reduce along one logical mesh axis, use `allReduceOnAxis`. pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const count = Compiler.current().meshes[0].numDevices(); - var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; - for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); - return allReduceGroups(inputs, func_, devices[0..count], count); -} - -/// Reduces local tensor values within process groups defined by `axis` in `mesh`. -/// A group contains processes with the same coordinates on every other logical -/// mesh axis. Separate replicas of the mesh belong to separate groups. Within -/// each group, `func_` combines corresponding elements and the result is identical -/// on all member processes. -/// -/// Use this inside a manual computation to combine partial results along `axis` -/// while preserving the independent computations along the other mesh axes. -/// -/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. -/// * `mesh` is the logical mesh used by the manual computation and must be -/// included in the compilation's meshes. -/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum -/// literal or a mesh axis tag. The axis must exist in `mesh`. -/// -/// Returns the same container type with tensor attributes as -/// `inputs`. An axis of size one returns `inputs` unchanged. -/// -/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local -/// expert contributions independently for each of the two data shards: -/// -/// ```zig -/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); -/// ``` -/// -/// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation`, an output spec such as -/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the -/// result replicated along `.experts`. -pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); - const size = mesh.grid.axis_sizes[logical_axis]; - var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; - var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - const replicas = mesh.numReplicas(); - for (0..mesh.numDevices()) |device| { - const coords = mesh.grid.logical_coords[device]; - var tile: usize = 0; - var group: usize = 0; - for (0..mesh.logical.axes.len) |dim| { - tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; - if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; - } - group = group * replicas + replica_indices[tile]; - replica_indices[tile] += 1; - groups[group * size + coords[logical_axis]] = @intCast(device); - } - return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); -} - -/// Reduces corresponding tensor elements independently within each process group. -/// `inputs`, `func_` and the return value follow `allReduce`'s contract. -/// -/// `groups` identifies processes by their executable partition IDs, as returned -/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. -/// -/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions -/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group -/// receive that group's result. Groups of size one return `inputs` unchanged. -fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -166,7 +75,8 @@ fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64 else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - if (group_size <= 1) return inputs; + const num_devices = ctx.meshes[0].numDevices(); + if (num_devices <= 1) return inputs; const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); @@ -215,9 +125,14 @@ fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64 break :b block; }; + var replica_group_storage: [constants.MAX_RANK]i64 = undefined; + for (0..@intCast(num_devices)) |i| { + replica_group_storage[i] = @intCast(i); + } + const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), - groups, + .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), + replica_group_slice, ); const handle = ctx.nextChannelId(); @@ -287,20 +202,6 @@ fn AllReduceReturnType(comptime InputsT: type) type { }; } -/// Returns the current process's zero based coordinate on `axis` in `mesh` -/// as a scalar u32. -/// -/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates -/// 0 and 1. Use this coordinate inside a manual computation to identify a shard -/// along one axis while `partitionId()` identifies a partition across the whole mesh. -pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); - var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; - for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; - return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) - .gather(.{ .device = partitionId() }, .{}); -} - pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); @@ -929,9 +830,10 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // Copy tags; the partitioner infers result sharding. + // copy tags and mesh info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; + new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } @@ -1532,15 +1434,7 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, - symmetric_workspace: ?SymmetricWorkspace = null, output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, - - /// Named per-device workspace shared across CuTe calls and executables. - /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. - pub const SymmetricWorkspace = struct { - name: []const u8, - bytes: i64, - }; }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1561,7 +1455,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1586,13 +1480,6 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); } - if (opts.symmetric_workspace) |workspace| { - stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); - attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ - .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), - .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), - }))); - } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = stablehlo.custom_call( @@ -2760,6 +2647,11 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, + /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies + /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, + /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output + /// sharding, this setting will avoid the shuffling of the buffer's data. + sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2793,7 +2685,23 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); + if (params.sharding_aware) { + return shardingAwareTypedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } else { + return typedCustomCall( + params.name, + opts, + input_tensors, + output_shapes, + attributes, + ); + } } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2821,6 +2729,44 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } +pub fn shardingAwareTypedCustomCall( + comptime target_name: [:0]const u8, + comptime opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + + var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); + } + + const Handler = struct { + input: Input, + attributes: Attributes, + + fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { + return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); + } + }; + const output_tensors = manualComputation( + Handler.body, + .{ .input = input, .attributes = attributes }, + @as([]const Shape, &output_shapes), + ); + + // Convert the slice back to a struct + var out: ShapeToTensor(Output) = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + @field(out, field_name) = output_tensors[i]; + } + return out; +} + pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2835,7 +2781,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2845,6 +2791,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2860,6 +2807,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2924,7 +2872,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2936,6 +2884,7 @@ pub fn typedCustomCall( } return out; }, + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -2988,7 +2937,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2999,17 +2948,9 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const Local = struct { - fn body(x: Tensor, output_shape: Shape) Tensor { - return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, - }); - } - }; - const output = manualComputation(Local.body, input, shape, tp, .{ - .in_specs = .parse(.{ ._0 = .model }), - .out_specs = .parse(.{ ._0 = .model }), + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index a0ec0037..c9596f98 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,131 +1,49 @@ const std = @import("std"); +const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); +const DataType = @import("../dtype.zig").DataType; const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; - const log = std.log.scoped(.@"zml/Compiler"); -/// Partitioning declarations for the tensors in `T`. -/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. -/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes -/// `struct { x: Logical, bias: ?Logical }`. +/// ZML operator describes computation across several devices, +/// but `manualComputation` allows to partition the devices in small group, +/// split the `Tensor` on those partitions, +/// and describes sub-computation that work inside a partition. /// -/// Initialize individual fields to declare different shardings, or use -/// `partitionSpecs` to apply one spec to every tensor in a value. -pub fn PartitionSpecs(comptime T: type) type { - return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); -} - -/// Declares the same sharding for a group of tensors at a `manualComputation` -/// boundary. +/// See: +/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop /// -/// ```zig -/// .in_specs = .{ -/// .query = .parse(.{ .h = .model }), -/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), -/// .metadata = partitionSpecs(metadata, .replicated), -/// } -/// ``` +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: +/// - `I` is user defined and contains inputs tensors and any metadata required by the computation +/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs +/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs +/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated +/// * `outputs` contains the global output shapes, +/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. /// -/// Requires an active compiler and uses its arena for allocations. Use -/// `partitionSpecsAlloc` outside graph construction. Use named struct fields -/// when mixing tensors with non-tensor metadata. -pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { - return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); -} - -/// Builds the same declarations as `partitionSpecs` using an explicit allocator. -pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { - var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; - try meta.mapAlloc(struct { - fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { - return spec_; - } - }.assign, allocator, spec, tensors, &specs); - return specs; -} - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = PartitionSpecs(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. +/// This depart from the rest of ZML where output shapes are generally inferred, +/// but in this case since `body_fn` explains how to compute parts of the result, +/// it's not clear how to put them back together, +/// hence why the user is asked to explicitly provided the expected output shape. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -136,42 +54,13 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); + var owned_inputs: Inputs = inputs; + const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + defer arena.free(flat_input_tensors); - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } + var owned_output_shapes: OutputShapes = output_shapes; + const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); + defer arena.free(flat_output_shapes); const WrappedInputs = struct { inputs: *const Inputs, @@ -183,17 +72,15 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; _ = manualComputationInternal( compiler, arena, - mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -203,7 +90,6 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; - return outputs_struct; } @@ -214,21 +100,27 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - global_output_shapes: []const Shape, + output_shapes: []*Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { + .mesh => |m| m, + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + }; + const manual_axes = collectManualAxes(mesh, input_tensors); + log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + const global_output_shapes = try arena.alloc(Shape, output_shapes.len); + for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -242,17 +134,11 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + const manual_block = mlir.Block.init(input_types, input_locs); + errdefer manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -264,14 +150,17 @@ noinline fn manualComputationInternal( }; } + for (output_shapes, global_output_shapes) |shape_ptr, global| { + shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { + stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -325,14 +214,97 @@ noinline fn manualComputationInternal( }, }; - // Returned tensors describe the operation's global output shapes and sharding. + // reuse local_outputs buffer to store the final result. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; + output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); } return local_outputs; } +const ChosenMesh = union(enum) { + mesh: *const Sharding.Mesh, + incompatible: [2]u32, +}; + +fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { + var mesh: ?*const Sharding.Mesh = null; + var mesh_source_id: u32 = @intCast(inputs.len); + for (0.., inputs) |i, input| { + if (input.shape()._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) + return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; + } else { + mesh = shape_mesh; + mesh_source_id = @intCast(i); + } + } + } + return .{ .mesh = mesh orelse fallback }; +} + +test chooseCommonMesh { + const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); + const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); + const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); + const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); + + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); +} + +fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { + var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); + for (inputs) |input| { + const sharding = input.shape()._sharding; + if (sharding.mesh == null) continue; + std.debug.assert(sharding.mesh.? == mesh); + for (0..input.rank()) |dim| { + if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; + } + } + + var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; + for (mesh.logical.axes.constSlice(), 0..) |axis, i| { + if (used[i]) axes.appendAssumeCapacity(axis); + } + return axes; +} + +test collectManualAxes { + const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); + var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); + var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); + + const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); + data = replicated; + model = replicated; + try std.testing.expectEqual(@as(usize, 2), axes.len); + try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); + try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); + + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const expected = shardy.ManualAxesAttr.get(ctx, &.{ + .init(ctx, "data"), + .init(ctx, "model"), + }).attribute(); + try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); +} + const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -375,172 +347,54 @@ const gspmd = struct { } }; -test manualComputation { - const zml = @import("../zml.zig"); +const shardy_ = struct { + pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { + var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; + for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); + return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); + } +}; + +test "manualComputation passes local output shapes to the callback" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); + const partitions = mesh.numPartitionsForLogicalAxis(.model); + if (partitions == 1) return error.SkipZigTest; - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, + var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); + defer compiler.deinit(); + compiler.activate(); + defer compiler.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = compiler.pushBlock(block); + defer scope.pop(); + + const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) + .withPartitioning(mesh, .{ .h = .model }); + const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); + const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); + var received_shape: ?Shape = null; + const Context = struct { + input: Tensor, + received_shape: *?Shape, fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); + self.received_shape.* = output_shape; + return self.input; } }; + const output = manualComputation( + Context.body, + .{ .input = input, .received_shape = &received_shape }, + global_shape, + ); - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} - -test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../zml.zig"); - const Logical = Sharding.Partitioning.Logical; - const Inputs = struct { - layers: []const Tensor, - bias: ?Tensor, - cache: union(enum) { disabled, values: [2]Tensor }, - scale: f32, - }; - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); - var inputs: Inputs = .{ - .layers = &.{ tensor, tensor }, - .bias = null, - .cache = .{ .values = .{ tensor, tensor } }, - .scale = 2, - }; - const heads: Logical = .parse(.{ .h = .model }); - const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); - const expected: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{ heads, heads }, - .bias = null, - .cache = .{ .values = .{ heads, heads } }, - }; - try std.testing.expectEqualDeep(expected, specs); - - inputs.layers = &.{}; - inputs.bias = tensor; - inputs.cache = .disabled; - const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); - const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{}, - .bias = .replicated, - .cache = .disabled, - }; - try std.testing.expectEqualDeep(expected_replicated, replicated); + try std.testing.expect(received_shape != null); + try std.testing.expect(received_shape.?.eql(local_shape)); + try std.testing.expect(output.shape().eql(global_shape)); + try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 824d2bd1..bfa4fc7d 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -14,7 +14,11 @@ const log = std.log.scoped(.@"zml/pjrtx"); pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; - pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { + /// Compiles on the calling thread: under zio, `Compiler.compile` has + /// already moved off the coroutine onto a pool worker of its own + /// (`zio.blockInPlaceReserved`), and XLA's compile needs the thread's + /// stack, not a task's. + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); @@ -37,7 +41,7 @@ pub const Client = opaque { }; }; - return @ptrCast(try client.compile(api, io, .{ + return @ptrCast(try client.compileRaw(api, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, diff --git a/zml/platform.zig b/zml/platform.zig index 54fb90e4..6f9d114d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,13 +3,12 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_rocm = @import("platforms/rocm"); const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; -const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); -const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -39,6 +38,10 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); return error.MissingDevices; } + if (num_devices > Platform.MAX_NUM_DEVICES) { + log.err("Platform {} got {} devices, but ZML supports up to {}", .{ target, num_devices, Platform.MAX_NUM_DEVICES }); + return error.InvalidDeviceCount; + } switch (target) { .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { if (!std.math.isPowerOfTwo(num_devices)) { @@ -259,12 +262,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, + fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, fn deinit(self: *CudaState) void { - if (self.flashinfer_cutlass_moe_runners) |runners| { + if (self.fi_cutlass_moe_runners) |runners| { runners.deinit(); - self.flashinfer_cutlass_moe_runners = null; + self.fi_cutlass_moe_runners = null; } } }; @@ -302,6 +305,16 @@ pub const Platform = struct { meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), replicated_mesh: *const zml.Mesh, io_impl: IoImpl, + /// Whether `releaseFreedDeviceMemory` asks the devices to give freed + /// memory back. `init` sets it for XLA's VMM allocator on CUDA and ROCm + /// (`CreateOptions.XlaGpu.Allocator.vmm`), which keeps freed memory + /// mapped, and counted in use, until it is reused or reclaimed. Null + /// otherwise: other allocators free at once, and oneAPI's VMM allocator + /// is not proven to follow the same rule. The flag lives in the + /// platform's arena, not in `Platform`: the first device that does not + /// answer as the VMM allocator does clears it for good, through a + /// `*const Platform`, without writing to the platform. + releases_freed_memory: ?*std.atomic.Value(bool) = null, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -315,11 +328,9 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + errdefer pjrt_client.deinit(api); const pjrt_devices = pjrt_client.addressableDevices(api); try validateDeviceCount(target, pjrt_devices.len); - if (pjrt_devices.len > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); - } const pjrt_memories = pjrt_client.addressableMemories(api); @@ -353,7 +364,18 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); + errdefer if (comptime Target.cuda.isEnabled()) platform.state.deinit(); try platform.meshes.ensureTotalCapacity(arena, 8); + const keeps_freed_memory = switch (target) { + .cuda => options.cuda.allocator == .vmm, + .rocm => options.rocm.allocator == .vmm, + else => false, + }; + if (keeps_freed_memory) { + const releases = try arena.create(std.atomic.Value(bool)); + releases.* = .init(true); + platform.releases_freed_memory = releases; + } { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -383,9 +405,16 @@ pub const Platform = struct { switch (target) { .cuda => { - cuda.initFlashinferCutlass(platform, io) catch |err| { - log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); - }; + if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { + zml.moe.cutlass_flashinfer.register(platform) catch |err| { + log.warn( + "Failed to register FlashInfer CUTLASS MoE custom calls: {}", + .{err}, + ); + }; + } else |err| { + log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); + } }, else => {}, } @@ -419,6 +448,117 @@ pub const Platform = struct { } else error.Unavailable; } + /// Forces lazy device allocators to materialize their backing pools. + /// + /// In particular, a preallocated GPU BFC allocator does not reserve its + /// fixed arena until its first non-empty allocation. The temporary buffer + /// is released immediately; caching allocators retain their initialized + /// pools for subsequent allocations. Calling this more than once is safe. + /// Devices are warmed concurrently: a serial pass over eight GPUs costs + /// measurable milliseconds ahead of a load. A device whose task `io` + /// cannot spawn is warmed in place. + pub fn warmupDeviceAllocators(self: *const Platform, io: std.Io) !void { + const Worker = struct { + platform: *const Platform, + device_index: usize, + first_error: *stdx.FirstError(pjrt.ApiError), + + fn run(worker: @This()) void { + const dims: []const i64 = &.{1}; + const memory = worker.platform.devices[worker.device_index].memory(.default).?; + const buffer = worker.platform.pjrt_client.createUninitializedBuffer(worker.platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = worker.platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = memory.pjrt_memory }, + }) catch |err| { + _ = worker.first_error.set(err); + return; + }; + buffer.deinit(worker.platform.pjrt_api); + } + }; + var first_error: stdx.FirstError(pjrt.ApiError) = .{}; + var group: std.Io.Group = .init; + for (self.devices, 0..) |_, device_index| { + const worker: Worker = .{ .platform = self, .device_index = device_index, .first_error = &first_error }; + group.concurrent(io, Worker.run, .{worker}) catch worker.run(); + } + try group.await(io); + if (first_error.get()) |err| return err; + } + + /// Gives back to the devices memory that nothing uses but that stays + /// held: the freed buffers XLA's VMM allocator keeps, which + /// `Device.memoryStats` counts in `bytes_in_use`, and on ROCm what the + /// HIP runtime keeps. + /// + /// XLA's VMM allocator keeps a freed allocation mapped, and counted in + /// use, until a request of the same size reuses it, a request beyond + /// the device's limit reclaims it, or an executable with remapped + /// command buffers is destroyed. A request one byte beyond the limit + /// completes the deferred frees of the device's memory, once its + /// compute stream reached them, then fails with `ResourceExhausted`, as + /// expected. It leaves out the frees of collective memory, which stay + /// counted, and of pinned host memory, which `bytes_in_use` does not + /// count. Any other answer means the plugin does not follow that rule: + /// it is logged once, naming the device, and clears + /// `releases_freed_memory`, so later calls skip this part. This part is + /// a no-op for other allocators, which free at once, and for a device + /// that reports no limit. + /// + /// On ROCm, whatever the allocator, it first makes the HIP runtime give + /// back the device memory it keeps outside the allocator, chiefly the + /// scratch memory of kernels that spill registers, which `memoryStats` + /// never counts but the devices do + /// (`platforms_rocm.releaseRuntimeDeviceMemory`). That part reaches + /// every queue the process has on these devices, other platforms' + /// included, and makes running work wait: each queue holding scratch is + /// suspended and resumed, the call waits for its kernels using scratch, + /// and allocations on the device stall meanwhile. A call within a load + /// can overlap the load's other compiles and transfers, which then wait + /// too. A missing HIP runtime, or a device that does not refuse, is + /// logged once and ends that part for the process. + pub fn releaseFreedDeviceMemory(self: *const Platform) void { + if (comptime Target.rocm.isEnabled()) { + if (self.target == .rocm) _ = self.releaseRocmRuntimeMemory(); + } + const releases = self.releases_freed_memory orelse return; + for (self.devices) |*device| { + if (!releases.load(.monotonic)) return; + const limit = device.memoryStats().bytes_limit orelse continue; + // A limit that large is no limit: nothing goes beyond it. + const beyond_limit = std.math.cast(i64, limit + 1) orelse continue; + const dims: []const i64 = &.{beyond_limit}; + const answer: []const u8 = if (self.pjrt_client.createUninitializedBufferSilent(self.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = self.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + })) |buffer| answer: { + buffer.deinit(self.pjrt_api); + break :answer "a buffer"; + } else |err| switch (err) { + error.ResourceExhausted => continue, + else => @errorName(err), + }; + if (releases.swap(false, .monotonic)) { + log.warn("Device {d} answered a request beyond its memory limit with {s}, not ResourceExhausted; freed device memory is no longer given back and may stay counted as in use", .{ device.id(), answer }); + } + return; + } + } + + /// `platforms_rocm.releaseRuntimeDeviceMemory` over every device: a + /// device's HIP ordinal is its local hardware id. + fn releaseRocmRuntimeMemory(self: *const Platform) bool { + var ordinals: [MAX_NUM_DEVICES]c_int = undefined; + for (self.devices, ordinals[0..self.devices.len]) |*device, *ordinal| { + ordinal.* = device.localHardwareId(); + } + return platforms_rocm.releaseRuntimeDeviceMemory(ordinals[0..self.devices.len]); + } + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; @@ -598,6 +738,14 @@ pub const Platform = struct { try writer.writeAll(" }"); } + /// `shape` placed as a buffer places it: on its mesh, or replicated on + /// every device (`replicated_mesh`) when it names none. + pub fn placedShape(self: *const Platform, shape: zml.Shape) zml.Shape { + var placed = shape; + if (placed._sharding.mesh == null) placed._sharding.mesh = self.replicated_mesh; + return placed; + } + pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { for (self.memories) |mem| { if (mem.isOfKind(kind)) { @@ -860,12 +1008,6 @@ pub const cuda = struct { } }; - /// SM count of an addressable CUDA device; unavailable attributes return null. - pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { - const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; - return std.math.cast(u32, attribute.int64); - } - pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { if (platform.target != .cuda) return null; const devices = platform.pjrt_client.devices(platform.pjrt_api); @@ -878,29 +1020,6 @@ pub const cuda = struct { } } else null; } - - fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { - if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; - const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; - - const allocator = platform.arena.allocator(); - const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); - errdefer allocator.destroy(runners); - - try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); - - runners.* = .{}; - errdefer runners.deinit(); - - try platform.registerFfi(.{ - .name = flashinfer_cutlass_contract.name, - .handler = flashinfer_cutlass_runtime.handler, - .traits = .{ .command_buffer_compatible = true }, - .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, - }); - - platform.state.cuda.flashinfer_cutlass_moe_runners = runners; - } }; pub const rocm = struct { @@ -1015,15 +1134,12 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const buffers = call_frame.args.buffers(); - if (buffers.len <= 0) return null; - const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = buffers[0]; + const buffer = call_frame.args.buffers()[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1047,13 +1163,9 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); - defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - - const event = pjrt_buffer.readyEvent(pjrt_api); - defer event.deinit(pjrt_api); - try event.awaitRaw(pjrt_api); + try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); @@ -1095,3 +1207,63 @@ test "platform defaultMemoryLayout is boring" { }); } } + +test "platform device allocators can be warmed repeatedly" { + const platform = zml.testing.env(); + + try platform.warmupDeviceAllocators(std.testing.io); + try platform.warmupDeviceAllocators(std.testing.io); +} + +test "platform ROCm devices refuse a request for their whole memory, then still work" { + if (comptime !Target.rocm.isEnabled()) return error.SkipZigTest; + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + // Every device refuses a request for its whole memory, the request ROCr + // answers by reclaiming its queues' scratch and freeing its caches. The + // release itself is not visible from here: HIP counts ROCr's caches as + // free, and only the driver's counters in sysfs show it. + try std.testing.expect(platform.releaseRocmRuntimeMemory()); + + // The devices still take and give back data afterwards. + const values = [_]u32{ 1, 2, 3, 4, 5, 6, 7, 8 }; + var buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, .init(.{ .n = values.len }, .u32), std.mem.sliceAsBytes(&values)); + defer buffer.deinit(); + const read: zml.Slice = try buffer.toSliceAlloc(std.testing.allocator, std.testing.io); + defer read.free(std.testing.allocator); + try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); +} + +test "platform gives freed device memory back" { + const platform = zml.testing.env(); + // On GPU CI, CUDA and ROCm run this with XLA's VMM allocator. + const releases = platform.releases_freed_memory orelse return error.SkipZigTest; + const device = &platform.devices[0]; + // Without a limit the drain does nothing. + try std.testing.expect(device.memoryStats().bytes_limit != null); + + // An odd size, so that no request of the same size reuses it once freed. + const size = 64 * 1024 * 1024 + 4096; + const dims: []const i64 = &.{size}; + const buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, .{ + .dims = dims, + .element_type = pjrtx.bufferTypeFromDtype(.u8), + .layout = platform.defaultMemoryLayout(dims, .u8), + .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, + }); + { + // Once its allocation is ready, destroying it frees it at once, + // not later on another thread. + errdefer buffer.deinit(platform.pjrt_api); + const ready = buffer.readyEvent(platform.pjrt_api); + defer ready.deinit(platform.pjrt_api); + try ready.await(platform.pjrt_api, std.testing.io); + } + buffer.deinit(platform.pjrt_api); + const in_use_freed = device.memoryStats().bytes_in_use; + + platform.releaseFreedDeviceMemory(); + try std.testing.expect(releases.load(.monotonic)); + try std.testing.expect(device.memoryStats().bytes_in_use + size <= in_use_freed); +} diff --git a/zml/quantization.zig b/zml/quantization.zig index db20619a..9053b26c 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -105,6 +105,38 @@ pub const Quantization = struct { return self == .mxfp8 or self == .mxfp4; } }; + + /// What a platform's FP8 GEMMs read, so block-FP8 weights can be bound + /// for it once. Binding them requires the store to carry one + /// (`zml.io.TensorStore.Options.quantization`); the code that binds them + /// reads it (`TensorStore.View.quantization`) and converts or lays them + /// out while binding. + pub const Target = struct { + /// The E4M3 encoding of FP8 weights and activations: FNUZ on CDNA3, + /// FN on CUDA, CDNA4 and RDNA4. Null on other platforms: block-FP8 + /// activations stay BF16 (`quantizeInput`), and weights keep the + /// checkpoint's encoding. + fp8: ?DataType = .f8e4m3fn, + /// Whether e8m0 block scales are laid out in the 128x4 blocks of + /// Blackwell's block-scaled MMA (CUDA sm_10x). + swizzled_scales: bool = false, + + /// `platform`'s target: on CUDA, FN, its scales swizzled on sm_10x; + /// on ROCm, by architecture, null when its compute capability is + /// unknown; elsewhere, CPU included, null and unswizzled. The only + /// platform switch for block-FP8 weights. + pub fn of(platform: *const Platform) Target { + return switch (platform.target) { + .cuda => .{ .swizzled_scales = if (platform_mod.cuda.computeCapability(platform)) |cc| cc.major == 10 else false }, + .rocm => .{ .fp8 = if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null }, + else => .{ .fp8 = null }, + }; + } + }; }; pub const QuantizedInput = struct { @@ -127,18 +159,7 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, .fp8_block32, .fp8_block128 => blk: { - const dtype: DataType = switch (platform.target) { - .cuda => .f8e4m3fn, - // RNGD contracts matching FP8 pairs in 128-wide K blocks. - .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, - } else return null, - else => return null, - }; - + const dtype = Quantization.Target.of(platform).fp8 orelse return null; break :blk switch (quantization.scheme) { .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), @@ -166,23 +187,6 @@ pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { }; } -pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { - const platform = zml.Compiler.current().platform; - - const dtype: ?DataType = switch (platform.target) { - .cuda, .cpu, .tpu => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null, - else => null, - }; - - if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); - return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); -} - fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { const bits = raw_scale.bitCast(.u32); return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 3827fe27..beee2cca 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,497 +2,19 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; +const testing = @import("testing.zig"); + +pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); const log = std.log.scoped(.@"zml/safetensors"); -// TODO: consider promoting this to the top level struct -// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights -/// Maps tensor names to their shape and positions in different sub files. -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, - - tensors: std.StringArrayHashMapUnmanaged(Tensor), - - /// Metadata found in `__metadata__` field of the safetensors header. - /// This is rarely use and the information there is not standardized, so we let user code handle it. - /// Here we just store the plain bytes, reach out if you need this to be already parsed. - metadata: []const u8, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = "", - }; - } - - pub fn deinit(self: *TensorRegistry) void { - self.arena.deinit(); - } - - pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { - const file_type, const file = try resolveModelEntrypoint(io, repo); - defer file.close(io); - - var registry: TensorRegistry = .init(allocator); - switch (file_type) { - .safetensors => try registry.parseSafetensors(io, allocator, file), - .index => try registry.parseIndexJson(io, allocator, repo, file), - .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), - .unknown => return error.InvalidPath, - } - return registry; - } - - pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { - const file_type: FileType = .resolve(path); - switch (file_type) { - .safetensors => { - // open directly - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - var registry: TensorRegistry = .init(allocator); - try registry.parseSafetensors(io, allocator, file); - return registry; - }, - .full_index, - .index, - => { - // resolve repo as parent folder then open - const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - - var registry: TensorRegistry = .init(allocator); - try switch (file_type) { - .index => registry.parseIndexJson(io, allocator, repo, file), - .full_index => registry.parseFullIndexJson(io, allocator, repo, file), - .safetensors, .unknown => unreachable, - }; - return registry; - }, - .unknown => { - // the path is possibly a folder, let's try that - const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return try fromRepo(allocator, io, repo); - }, - } - } - - /// Register a tensor, duplicate tensor name and uri. - /// If you create a lot of tensors sharing the same uri, - /// considere calling `registry.tensors` directly and avoid duplicated uris. - pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { - const allocator = self.arena.allocator(); - - const owned_name = try allocator.dupe(u8, tensor.name); - errdefer allocator.free(owned_name); - const owned_uri = try allocator.dupe(u8, tensor.file_uri); - errdefer allocator.free(owned_uri); - - const owned_tensor: Tensor = .{ - .name = owned_name, - .file_uri = owned_uri, - .shape = tensor.shape, - .offset = tensor.offset, - }; - try self.tensors.put(allocator, owned_name, owned_tensor); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } - - /// Parse a given .safetensors file and create entries for each tensor - pub fn parseSafetensors( - self: *TensorRegistry, - io: std.Io, - scratch_allocator: std.mem.Allocator, - file: std.Io.File, - ) !void { - var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); - defer tmp_state.deinit(); - const tmp_arena = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); - const file_name_len = try file.realPath(io, owned_file_name); - // Try to release extra memory, but we don't care if it fails - _ = self_arena.resize(owned_file_name, file_name_len); - owned_file_name = owned_file_name[0..file_name_len]; - - const header_buf = try tmp_arena.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = file.reader(io, header_buf); - - const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); - const BYTES_HEADER = 8; - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); - - try file_reader.interface.readSliceAll(json_data); - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - tmp_arena, - json_data, - .{}, - ); - - try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); - for (headers.object.keys(), headers.object.values()) |key, value| { - if (std.mem.eql(u8, key, "__metadata__")) { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - continue; - } - - const owned_name = try self_arena.dupe(u8, key); - const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); - self.tensors.putAssumeCapacity(owned_name, tensor); - } - } - - pub fn parseIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; - try safetensors_files.ensureUnusedCapacity(tmp, 64); - if (index.object.get("weight_map")) |wm| { - for (wm.object.values()) |file_name| { - try safetensors_files.put(tmp, file_name.string, {}); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - for (safetensors_files.keys()) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try self.parseSafetensors(io, tmp, file); - } - } - - pub fn parseFullIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - - var repo_uri_buf = try tmp.alloc(u8, 1024); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - - // Note: this is relatively inefficient, we read the full file, copy most of it's content, - // then reduce to TensorRegistry format. - // We could update TensorRegistry as we stream the json. - // Zig std json API are a bit awkward around streaming though. - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try realpaths.ensureUnusedCapacity(tmp, 64); - const wm = index.object.get("weight_map") orelse return error.InvalidJson; - - try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); - for (wm.object.keys(), wm.object.values()) |key, value| { - const tensor_name = try self_arena.dupe(u8, key); - const object = value.object; - const filename = value.object.get("filename").?.string; - const filename_entry = try realpaths.getOrPut(tmp, filename); - const file_uri = if (filename_entry.found_existing) - filename_entry.value_ptr.* - else uri: { - const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); - filename_entry.value_ptr.* = uri; - break :uri uri; - }; - - const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); - self.tensors.putAssumeCapacity(tensor_name, tensor_info); - } - - if (index.object.get("metadata")) |metadata| { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - } - } - - pub fn saveAsFullIndexJson( - self: *const TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - out: *std.Io.Writer, - ) !void { - var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); - defer gpa.free(repo_uri_buf); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - var total_size: u64 = 0; - for (self.tensors.values()) |value| total_size += value.byteSize(); - - try out.print( - \\{{ - \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, - \\"weight_map": {{ - \\ - , .{ self.tensors.count(), total_size }); - - const n = self.tensors.count(); - for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { - std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); - const local_name = value.file_uri[repo_name.len + 1 ..]; - const end_offset = value.offset + value.byteSize(); - const safetensors_dtype: []const u8 = dt: { - for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { - if (zml_dt == value.shape._dtype) break :dt st_dt; - } - // Note: we could bitcast to u8, - // but reaching this panic implies the user didn't loaded this TensorRegistry - // from a safetensors file, and manually created the dict. - // In which case, they know what they are doing, - // and can easily insert the bitcast themselves. - std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); - }; - const comma = if (i + 1 == n) "" else ","; - try out.print( - \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} - \\ - , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); - } - - try out.writeAll( - \\} - \\} - \\ - ); - try out.flush(); - } -}; - -pub const TensorReader = struct { - tensor: Tensor, - file: std.Io.File, - file_reader: std.Io.File.Reader, - alignment: ?std.mem.Alignment, - remaining: u64, - padding_remaining: u64, - - io: std.Io, - interface: std.Io.Reader, - - pub const InitOpts = struct { - alignment: ?std.mem.Alignment = null, - }; - - pub fn init( - io: std.Io, - tensor: Tensor, - buffer: []u8, - opts: InitOpts, - ) !TensorReader { - const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); - errdefer file.close(io); - - const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { - const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); - break :blk .{ aligned_offset, tensor.offset - aligned_offset }; - } else blk: { - break :blk .{ tensor.offset, 0 }; - }; - - var file_reader = file.reader(io, buffer); - try file_reader.seekTo(offset); - - return .{ - .tensor = tensor, - .file = file, - .file_reader = file_reader, - .alignment = opts.alignment, - .remaining = tensor.byteSize(), - .padding_remaining = padding_remaining, - .io = io, - .interface = .{ - .vtable = if (opts.alignment) |_| blk: { - break :blk &.{ - .stream = streamAligned, - .discard = discardAligned, - }; - } else blk: { - break :blk &.{ - .stream = stream, - .discard = discard, - }; - }, - .buffer = &.{}, - .seek = 0, - .end = 0, - }, - }; - } - - pub fn deinit(self: *TensorReader) void { - self.file.close(self.io); - } - - // Limit single reads to 1GB to avoid issues on macOS where pread returns - // INT_MAX when the requested size is larger than 2GB - const max_single_read_bytes: u64 = 1 << 30; - - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); - const n = try self.file_reader.interface.stream(w, combined_limit); - self.remaining -= n; - return n; - } - - fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } - - fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const align_bytes = self.alignment.?.toByteUnits(); - const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); - const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; - const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); - - if (file_offset != writer_offset) { - const skip = if (file_offset >= writer_offset) blk: { - break :blk file_offset - writer_offset; - } else blk: { - break :blk align_bytes - writer_offset + file_offset; - }; - - w.advance(skip); - } - - if (w.writableSliceGreedy(align_bytes)) |dest| { - const phys_buf = dest.ptr - file_offset; - const phys_pos = self.file_reader.pos - file_offset; - const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); - - if (phys_size > file_offset) { - const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; - if (phys_read > file_offset) { - const bytes_read = phys_read - file_offset; - w.advance(bytes_read); - self.file_reader.pos += bytes_read; - self.remaining -= bytes_read; - return bytes_read; - } - } - } else |_| {} - - const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); - self.remaining -= bytes_read; - - return bytes_read; - } - - fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return 0; - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } -}; +const BYTES_HEADER = 8; +/// A checkpoint tensor: its name, its shape, and where its bytes are, at +/// `offset` of the file `file_uri` names. The URI is what the io that read +/// the header resolved, a VFS URI under a VFS, so read the tensor through +/// an io that resolves it alike. A registry's entries own nothing; the +/// registry owns their strings. pub const Tensor = struct { file_uri: []const u8, name: []const u8, @@ -503,32 +25,31 @@ pub const Tensor = struct { return self.shape.byteSize(); } - pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { - return TensorReader.init(io, self, buffer, opts); + /// Reads the whole tensor into `buffer`, which holds exactly its bytes + /// (`OutOfBounds` otherwise). Opens `file_uri` through `io` for the + /// call, failing as that io's open does; a file that ends inside the + /// tensor is `UnexpectedEndOfFile`. + pub fn read(self: Tensor, io: std.Io, buffer: []u8) !void { + if (buffer.len != self.byteSize()) return error.OutOfBounds; + const file = try std.Io.Dir.openFile(.cwd(), io, self.file_uri, .{ .mode = .read_only }); + defer file.close(io); + return self.readRange(io, file, buffer, 0); } - pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { - const shape_field = object.get("shape").?.array; - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); - return error.TooManyDimensions; - } + pub const RangeError = std.Io.File.ReadPositionalError || error{ OutOfBounds, UnexpectedEndOfFile }; - const offset_field = object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(object.get("dtype").?.string); - - var dims: Shape.DimsArray = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - return .{ - .file_uri = file_uri, - .name = name, - .shape = .init(dims.slice(), dtype), - .offset = base_offset + start, - }; + /// Reads `buffer.len` bytes of the tensor from `tensor_offset`, relative + /// to its data, out of `file`, which the caller opened at `file_uri`. A + /// range past the tensor is `OutOfBounds`, a file that ends inside it + /// `UnexpectedEndOfFile`. The reads are positional, so concurrent calls + /// may share one file. + pub fn readRange(self: Tensor, io: std.Io, file: std.Io.File, buffer: []u8, tensor_offset: u64) RangeError!void { + const tensor_size = self.byteSize(); + if (tensor_offset > tensor_size or buffer.len > tensor_size - tensor_offset) return error.OutOfBounds; + if (buffer.len == 0) return; + const file_offset = std.math.add(u64, self.offset, tensor_offset) catch return error.OutOfBounds; + _ = std.math.add(u64, file_offset, buffer.len) catch return error.OutOfBounds; + if (try file.readPositionalAll(io, buffer, file_offset) != buffer.len) return error.UnexpectedEndOfFile; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -542,100 +63,727 @@ pub const Tensor = struct { } }; -pub const FileType = enum { - full_index, - index, - safetensors, - unknown, +test "Tensor.readRange reads tensor-relative ranges within the tensor and its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); - pub fn resolve(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { - return .full_index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + defer file.close(io); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var tensor: Tensor = .{ + .file_uri = "unused", + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var middle: [4]u8 = undefined; + try tensor.readRange(io, file, &middle, 3); + try std.testing.expectEqualStrings("3456", &middle); + + var empty: [0]u8 = .{}; + try tensor.readRange(io, file, &empty, 10); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &empty, 11)); + + var too_large: [2]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &too_large, 9)); + + // A file that ends inside the tensor. + tensor.shape = .init(.{12}, .u8); + var tail: [3]u8 = undefined; + try std.testing.expectError(error.UnexpectedEndOfFile, tensor.readRange(io, file, &tail, 9)); + + tensor.offset = std.math.maxInt(u64); + var overflow: [1]u8 = undefined; + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 0)); + try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 1)); +} + +test "Tensor.read reads the whole tensor from its file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); + try file.writePositionalAll(io, "prefix0123456789", 0); + + var path_buffer: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buffer); + file.close(io); + + const tensor: Tensor = .{ + .file_uri = path_buffer[0..path_len], + .name = "tensor", + .shape = .init(.{10}, .u8), + .offset = "prefix".len, + }; + var whole: [10]u8 = undefined; + try tensor.read(io, &whole); + try std.testing.expectEqualStrings("0123456789", &whole); + try std.testing.expectError(error.OutOfBounds, tensor.read(io, whole[0..9])); +} + +/// The tensors of one or more checkpoints by name, read from their +/// safetensors headers; no tensor data is read. It owns every name and URI. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: Tensors, + + /// An empty registry. + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + }; + } + + /// The checkpoint in `repo`: its `model.safetensors.index.json` and the + /// files it lists, else `model.safetensors`. `repo` stays the caller's + /// to close. + pub fn fromRepo( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + ) !TensorRegistry { + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; + } + + /// The checkpoint at `path`, read through `io`: a `.safetensors` file, + /// a `.safetensors.index.json` and the files it lists, or a directory + /// holding `model.safetensors.index.json` or `model.safetensors`. A + /// relative path is from the cwd; under a VFS, `path` may be a remote + /// URI. A missing path or entrypoint is `FileNotFound`, a denied + /// directory or entrypoint `PermissionDenied`; a file opened by name, + /// the one given or one an index lists, fails as `io`'s open does (a + /// denied local file is `AccessDenied`). A malformed header, such as + /// one whose offsets leave the file in a truncated download or one + /// shorter than its 8-byte length field, or an index without a + /// `weight_map` from tensor names to file names, is + /// `InvalidSafetensorsHeader`, and a name two of its files repeat + /// `DuplicateTensor`, both logged. + pub fn fromPath( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, + ) !TensorRegistry { + const repo_path, const file_kind = locateCheckpoint(path); + var repo = try resolveModelRepo(io, repo_path); + defer repo.close(io); + const kind = file_kind orelse return fromRepo(allocator, io, repo); + + // `repo` is the file's directory: the file opens by its base name. + const entrypoint = try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }); + defer entrypoint.close(io); + + var registry: TensorRegistry = .init(allocator); + errdefer registry.deinit(); + try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); + return registry; + } + + /// Adds every tensor of the checkpoint in `repo`, read as `fromRepo` + /// reads it, under `name` and a dot: `View.withPrefix(name)` reaches + /// them. A second checkpoint, such as a speculative drafter's, joins + /// the registry under a name of its own, and one store then holds both. + /// Adding under a name that a tensor already here is named under fails + /// with `error.DuplicateTensor`, logged, before anything is read. `repo` + /// stays the caller's to close. + pub fn addRepo(self: *TensorRegistry, io: std.Io, repo: std.Io.Dir, name: []const u8) !void { + const prefix = try std.mem.concat(self.arena.child_allocator, u8, &.{ name, "." }); + defer self.arena.child_allocator.free(prefix); + // A checkpoint added beside another owns its prefix: no tensor already + // here may be named under it. + for (self.tensors.keys()) |key| { + if (std.mem.startsWith(u8, key, prefix)) { + log.warn("Cannot add a checkpoint under {s}: tensor {s} is already named under it", .{ name, key }); + return error.DuplicateTensor; + } } + const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + try addCheckpoint(io, self, repo, entrypoint, kind, prefix); + } + + /// Frees every entry, its name and URI. + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.arena.deinit(); + } + + /// Adds `tensor`, its name and file URI copied; a name already here is + /// `DuplicateTensor`, logged. + pub fn registerTensor( + self: *TensorRegistry, + tensor: Tensor, + ) !void { + const allocator = self.arena.allocator(); + + var tensor_copy = tensor; + + tensor_copy.name = try allocator.dupe(u8, tensor.name); + tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); + + const entry = try self.tensors.getOrPut(allocator, tensor_copy.name); + if (entry.found_existing) { + log.warn("Tensor {s} of {s} is already registered from {s}", .{ tensor.name, tensor.file_uri, entry.value_ptr.file_uri }); + return error.DuplicateTensor; + } + entry.value_ptr.* = tensor_copy; } }; +/// What a checkpoint's entrypoint is: an index of the safetensors files +/// holding its tensors, or the one file holding them. +const EntrypointKind = enum { index, safetensors }; + +/// Where the checkpoint at `path` is (`TensorRegistry.fromPath`): its +/// directory, and the kind of the file `path` names there, or null when +/// `path` names the directory itself. +fn locateCheckpoint(path: []const u8) struct { []const u8, ?EntrypointKind } { + const kind: ?EntrypointKind = if (std.mem.endsWith(u8, path, ".safetensors.index.json")) + .index + else if (std.mem.endsWith(u8, path, ".safetensors")) + .safetensors + else + null; + return .{ if (kind == null) path else std.fs.path.dirname(path) orelse ".", kind }; +} + +/// Adds the tensors of the checkpoint whose entrypoint is open in `repo` to +/// `registry` under `prefix`. An index's files are read from `repo`. +fn addCheckpoint( + io: std.Io, + registry: *TensorRegistry, + repo: std.Io.Dir, + entrypoint: std.Io.File, + kind: EntrypointKind, + prefix: []const u8, +) !void { + switch (kind) { + .safetensors => return parseSafetensors(io, registry, entrypoint, prefix), + .index => { + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + var index_uri_buf: [1024]u8 = undefined; + const index_uri = index_uri_buf[0..try entrypoint.realPath(io, &index_uri_buf)]; + + var index_reader = entrypoint.reader(io, try scratch.alloc(u8, 64 * 1024)); + const file_names = indexFiles(scratch, &index_reader.interface) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.ReadFailed => return index_reader.err orelse err, + else => { + log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + for (file_names) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try parseSafetensors(io, registry, file, prefix); + } + }, + } +} + +/// The files a safetensors index maps tensors to, each once, in the order +/// its `weight_map` first names them, allocated in `arena`. An index +/// without a `weight_map` from tensor names to file names fails with the +/// JSON parser's error, unlogged. +fn indexFiles(arena: std.mem.Allocator, index: *std.Io.Reader) ![]const []const u8 { + var json_reader: std.json.Reader = .init(arena, index); + // The default allocation, `.alloc_always` over a `json.Reader`, copies + // every string out of `index`'s buffer, which the next refill overwrites. + const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; + const parsed = try std.json.parseFromTokenSourceLeaky(Index, arena, &json_reader, .{ .ignore_unknown_fields = true }); + var files: std.StringArrayHashMapUnmanaged(void) = .empty; + for (parsed.weight_map.map.values()) |file_name| try files.put(arena, file_name, {}); + return files.keys(); +} + +/// The safetensors files `TensorRegistry.fromPath` reads for `path`, named +/// in their directory, which is returned open for the caller to close: the +/// file a `.safetensors` path names, the files a `.safetensors.index.json` +/// lists, else those the directory's `model.safetensors.index.json` lists, +/// or its `model.safetensors`. The names are allocated in `arena`. No +/// header is read and nothing is logged: a failure here is one `fromPath` +/// reports, with its context. +pub fn checkpointFiles(arena: std.mem.Allocator, io: std.Io, path: []const u8) !struct { std.Io.Dir, []const []const u8 } { + const repo_path, const file_kind = locateCheckpoint(path); + const repo = try std.Io.Dir.openDir(.cwd(), io, repo_path, .{}); + errdefer repo.close(io); + const entrypoint, const kind, const name = if (file_kind) |kind| + .{ try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }), kind, std.fs.path.basename(path) } + else + try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + const names = switch (kind) { + .safetensors => try arena.dupe([]const u8, &.{name}), + .index => names: { + var index_reader = entrypoint.reader(io, try arena.alloc(u8, 64 * 1024)); + break :names indexFiles(arena, &index_reader.interface) catch |err| switch (err) { + error.ReadFailed => return index_reader.err orelse err, + else => |e| return e, + }; + }, + }; + return .{ repo, names }; +} + +/// Adds the tensors of the safetensors `file` to `registry`, each named +/// `prefix` followed by its key in the header. A header that is not an +/// object of tensors, each with a `dtype`, a `shape` and `data_offsets` +/// spanning exactly its bytes within the file, is +/// `InvalidSafetensorsHeader`, and a name already registered +/// `DuplicateTensor`, both logged. +pub fn parseSafetensors( + io: std.Io, + registry: *TensorRegistry, + file: std.Io.File, + prefix: []const u8, +) !void { + var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); + defer arena.deinit(); + + var reader = file.reader(io, try arena.allocator().alloc(u8, 64 * 1024)); + + var file_uri_buf: [1024]u8 = undefined; + const file_uri_len = try file.realPath(io, &file_uri_buf); + const file_uri = file_uri_buf[0..file_uri_len]; + + // `File.Reader` returns `ReadFailed` with its cause in `err`, which a + // failed seek leaves unset. + const json_header_length: u64 = reader.interface.takeInt(u64, .little) catch |err| switch (err) { + error.EndOfStream => { + log.warn("Invalid safetensors header in {s}: shorter than its {d}-byte header length", .{ file_uri, BYTES_HEADER }); + return error.InvalidSafetensorsHeader; + }, + error.ReadFailed => return reader.err orelse err, + }; + // The header and every tensor's data must lie inside the file. + const file_length = try file.length(io); + if (json_header_length > file_length -| BYTES_HEADER) { + log.warn("Invalid safetensors header in {s}: {d} bytes of header in a file of {d}", .{ file_uri, json_header_length, file_length }); + return error.InvalidSafetensorsHeader; + } + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); + + reader.interface.readSliceAll(json_data) catch |err| return switch (err) { + error.ReadFailed => reader.err orelse err, + error.EndOfStream => err, + }; + + const headers = std.json.parseFromSliceLeaky(std.json.ArrayHashMap(std.json.Value), arena.allocator(), json_data, .{}) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: not a JSON object ({t})", .{ file_uri, err }); + return error.InvalidSafetensorsHeader; + }, + }; + const data_len = file_length - data_start_offset; + + // `data_offsets` are `[start, end)` after the header. + const Entry = struct { dtype: []const u8, shape: []const i64, data_offsets: [2]u64 }; + var it = headers.map.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; + + // Free-form strings, which nothing reads. + if (std.mem.eql(u8, key, "__metadata__")) continue; + + const entry = std.json.parseFromValueLeaky(Entry, arena.allocator(), kv.value_ptr.*, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + else => { + log.warn("Invalid safetensors header in {s}: tensor {s} needs a string dtype, an integer shape and two unsigned data_offsets ({t})", .{ file_uri, key, err }); + return error.InvalidSafetensorsHeader; + }, + }; + + if (entry.shape.len > Shape.MAX_RANK) { + log.warn("Cannot load {s}: tensor {s} has {d} dims, more than {d}", .{ file_uri, key, entry.shape.len, Shape.MAX_RANK }); + return error.TooManyDimensions; + } + const dtype = stringToDtype(entry.dtype) orelse { + log.warn("Cannot load {s}: tensor {s} has dtype {s}, which zml does not support", .{ file_uri, key, entry.dtype }); + return error.UnsupportedDataType; + }; + + const start, const end = entry.data_offsets; + const byte_size = checkedByteSize(entry.shape, dtype); + if (byte_size == null or start > end or end > data_len or end - start != byte_size.?) { + log.warn("Invalid safetensors header in {s}: tensor {s} has data_offsets [{d},{d}], which must span the bytes of its {s} shape {any} within the file's {d} data bytes", .{ file_uri, key, start, end, entry.dtype, entry.shape, data_len }); + return error.InvalidSafetensorsHeader; + } + + const tensor: Tensor = .{ + .file_uri = file_uri, + .name = if (prefix.len == 0) key else try std.mem.concat(arena.allocator(), u8, &.{ prefix, key }), + .shape = .init(entry.shape, dtype), + .offset = data_start_offset + start, + }; + + try registry.registerTensor(tensor); + } +} + +/// The bytes of `dims` of `dtype`, as `Shape.byteSize` counts them, or null +/// on a negative dim or a bit count past u64. +fn checkedByteSize(dims: []const i64, dtype: DataType) ?u64 { + var bits: u64 = dtype.bitSizeOf(); + for (dims) |d| bits = std.math.mul(u64, bits, std.math.cast(u64, d) orelse return null) catch return null; + return std.math.divCeil(u64, bits, 8) catch unreachable; +} + +test "parseSafetensors rejects a malformed header, data_offsets that do not span their tensor within the file, or a name already registered" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const max = "18446744073709551615"; + // One U8 tensor over the file's 4 data bytes. `header` overrides the + // whole header, `header_len` the header's length field. + const cases = [_]struct { offsets: []const u8 = "[0,4]", shape: []const u8 = "[4]", header: ?[]const u8 = null, header_len: ?u64 = null, valid: bool = false }{ + .{ .valid = true }, + .{ .offsets = "[4,0]" }, + .{ .offsets = "[0,3]" }, + .{ .offsets = "[0,5]" }, + .{ .offsets = "[-4,4]" }, + .{ .offsets = "[0," ++ max ++ "]" }, + .{ .offsets = "[" ++ max ++ "," ++ max ++ "]" }, + .{ .offsets = "[0,18446744073709551616]" }, + .{ .offsets = "[0]" }, + .{ .shape = "[-4]" }, + .{ .shape = "[4294967296,4294967296]" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":[4]}}" }, + .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":4,\"data_offsets\":[0,4]}}" }, + .{ .header = "[]" }, + .{ .header = "{\"x\":" }, + .{ .header_len = 1 << 20 }, + .{ .header_len = std.math.maxInt(u64) }, + }; + for (cases, 0..) |case, index| { + var header_buffer: [128]u8 = undefined; + const header = case.header orelse try std.fmt.bufPrint(&header_buffer, "{{\"x\":{{\"dtype\":\"U8\",\"shape\":{s},\"data_offsets\":{s}}}}}", .{ case.shape, case.offsets }); + var name_buffer: [32]u8 = undefined; + const file = try tmp.dir.createFile(io, try std.fmt.bufPrint(&name_buffer, "{d}.safetensors", .{index}), .{ .read = true }); + defer file.close(io); + var length: [BYTES_HEADER]u8 = undefined; + std.mem.writeInt(u64, &length, case.header_len orelse header.len, .little); + try file.writePositionalAll(io, &length, 0); + try file.writePositionalAll(io, header, length.len); + try file.writePositionalAll(io, &.{ 1, 2, 3, 4 }, length.len + header.len); + + var registry: TensorRegistry = .init(allocator); + defer registry.deinit(); + const result = parseSafetensors(io, ®istry, file, ""); + if (case.valid) { + try result; + try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); + // A name already registered, as one repeated across a checkpoint's + // shards. + try std.testing.expectError(error.DuplicateTensor, parseSafetensors(io, ®istry, file, "")); + } else { + try std.testing.expectError(error.InvalidSafetensorsHeader, result); + } + } +} + +test "addRepo adds a second checkpoint under a prefix" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // The target is one file, the drafter an index over one shard; both name + // their tensor `w`. + const target = try tmp.dir.createDirPathOpen(io, "target", .{}); + defer target.close(io); + try testing.writeSafetensorsFile(allocator, io, target, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + const drafter = try tmp.dir.createDirPathOpen(io, "drafter", .{}); + defer drafter.close(io); + try testing.writeSafetensorsFile(allocator, io, drafter, "shard.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 5, 6, 7, 8 } }, + }); + try drafter.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"shard.safetensors\"}}" }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, target); + defer registry.deinit(); + try registry.addRepo(io, drafter, "drafter"); + try std.testing.expectError(error.DuplicateTensor, registry.addRepo(io, drafter, "drafter")); + + try std.testing.expectEqual(@as(usize, 2), registry.tensors.count()); + const cases = [_]struct { name: []const u8, bytes: [4]u8 }{ + .{ .name = "w", .bytes = .{ 1, 2, 3, 4 } }, + .{ .name = "drafter.w", .bytes = .{ 5, 6, 7, 8 } }, + }; + for (cases) |case| { + const tensor = registry.tensors.get(case.name).?; + try std.testing.expectEqualStrings(case.name, tensor.name); + var bytes: [4]u8 = undefined; + try tensor.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &case.bytes, &bytes); + } +} + +test "fromPath opens a file, an index or a directory by a path relative to the cwd" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "layer.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, + }); + for ([_][]const u8{ "layer.safetensors.index.json", "model.safetensors.index.json" }) |index| { + try tmp.dir.writeFile(io, .{ .sub_path = index, .data = "{\"weight_map\":{\"w\":\"layer.safetensors\"}}" }); + } + + for ([_][]const u8{ "layer.safetensors", "layer.safetensors.index.json", "." }) |name| { + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var bytes: [4]u8 = undefined; + try registry.tensors.get("w").?.read(io, &bytes); + try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); + } +} + +test "fromPath refuses a file shorter than its header length or an index without a weight_map" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const files = [_]struct { []const u8, []const u8 }{ + .{ "short.safetensors", &.{ 1, 2, 3 } }, + .{ "list.safetensors.index.json", "[]" }, + .{ "unmapped.safetensors.index.json", "{\"metadata\":{\"total_size\":4}}" }, + .{ "unnamed.safetensors.index.json", "{\"weight_map\":{\"w\":4}}" }, + }; + for (files) |file| { + const name, const data = file; + try tmp.dir.writeFile(io, .{ .sub_path = name, .data = data }); + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + try std.testing.expectError(error.InvalidSafetensorsHeader, TensorRegistry.fromPath(allocator, io, path)); + } +} + +test "fromRepo reads every file an index longer than its read buffer names" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // About 90 KiB of weight map, read through a 64 KiB buffer: file names + // straddle refills. + const shards = 16; + const tensors_per_shard = 64; + var index: std.Io.Writer.Allocating = .init(scratch); + try index.writer.writeAll("{\"metadata\":{\"total_size\":1024},\"weight_map\":{"); + for (0..shards) |shard| { + const file_name = try std.fmt.allocPrint(scratch, "model-{d:0>5}-of-{d:0>5}.safetensors", .{ shard + 1, shards }); + var tensors: [tensors_per_shard]testing.TensorBytes = undefined; + for (&tensors, 0..) |*tensor, expert| { + const name = try std.fmt.allocPrint(scratch, "model.layers.{d}.mlp.experts.{d}.down_proj.weight", .{ shard, expert }); + tensor.* = .{ .name = name, .dtype = "U8", .dims = &.{1}, .bytes = &.{0} }; + try index.writer.print("{s}\"{s}\":\"{s}\"", .{ if (shard + expert == 0) "" else ",", name, file_name }); + } + try testing.writeSafetensorsFile(allocator, io, tmp.dir, file_name, &tensors); + } + try index.writer.writeAll("}}"); + try std.testing.expect(index.written().len > 64 * 1024); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = index.written() }); + + var registry: TensorRegistry = try .fromRepo(allocator, io, tmp.dir); + defer registry.deinit(); + // Each file holds its own tensors: a misread file name would lose some. + try std.testing.expectEqual(@as(usize, shards * tensors_per_shard), registry.tensors.count()); +} + +test "checkpointFiles names the files fromPath reads" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const scratch = arena.allocator(); + + // The index names a twice and leaves out c, which sits beside its files + // as a Mistral-format `consolidated.safetensors` does. + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "a.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{1} }, + .{ .name = "x", .dtype = "U8", .dims = &.{1}, .bytes = &.{2} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "b.safetensors", &.{ + .{ .name = "y", .dtype = "U8", .dims = &.{1}, .bytes = &.{3} }, + }); + try testing.writeSafetensorsFile(allocator, io, tmp.dir, "c.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{4} }, + }); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"a.safetensors\",\"y\":\"b.safetensors\",\"x\":\"a.safetensors\"}}" }); + const single = try tmp.dir.createDirPathOpen(io, "single", .{}); + defer single.close(io); + try testing.writeSafetensorsFile(allocator, io, single, "model.safetensors", &.{ + .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{5} }, + }); + + const cases = [_]struct { []const u8, []const []const u8 }{ + .{ ".", &.{ "a.safetensors", "b.safetensors" } }, + .{ "model.safetensors.index.json", &.{ "a.safetensors", "b.safetensors" } }, + .{ "c.safetensors", &.{"c.safetensors"} }, + .{ "single", &.{"model.safetensors"} }, + }; + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + for (cases) |case| { + const name, const expected = case; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); + const repo, const files = try checkpointFiles(scratch, io, path); + defer repo.close(io); + try std.testing.expectEqual(expected.len, files.len); + for (expected, files) |expected_file, file| { + try std.testing.expectEqualStrings(expected_file, file); + (try repo.openFile(io, file, .{})).close(io); + } + // The registry reads these files and no other. + var registry: TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + var read: std.StringArrayHashMapUnmanaged(void) = .empty; + for (registry.tensors.values()) |tensor| try read.put(scratch, std.fs.path.basename(tensor.file_uri), {}); + try std.testing.expectEqual(read.count(), files.len); + for (files) |file| try std.testing.expect(read.contains(file)); + } + const missing = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/missing", .{&tmp.sub_path}); + try std.testing.expectError(error.FileNotFound, checkpointFiles(scratch, io, missing)); +} + pub const ModelPathResolutionError = error{ FileNotFound, + PermissionDenied, InvalidPath, -} || std.mem.Allocator.Error; + Canceled, +}; -pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { - switch (FileType.resolve(path)) { - .full_index, .index, .safetensors => { - // The path is a file, let's open it's folder - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; +/// Opens the model directory at `path`, relative to the cwd. A path that is +/// not a directory, such as a file's, is `InvalidPath`, logged. +pub fn resolveModelRepo( + io: std.Io, + path: []const u8, +) ModelPathResolutionError!std.Io.Dir { + return std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |err| switch (err) { + error.FileNotFound => error.FileNotFound, + error.AccessDenied, error.PermissionDenied => error.PermissionDenied, + error.Canceled => error.Canceled, + else => { + log.warn("{s} is not a model directory: {t}", .{ path, err }); + return error.InvalidPath; }, - .unknown => { - // The path is probably a directory, let's try to open it as such - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return dir; - }, - } -} - -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { - const default_entrypoints: []const struct { FileType, []const u8 } = &.{ - .{ .full_index, "model.safetensors.full_index.json" }, - .{ .full_index, "consolidated.safetensors.full_index.json" }, - .{ .index, "model.safetensors.index.json" }, - .{ .index, "consolidated.safetensors.index.json" }, - .{ .safetensors, "model.safetensors" }, - .{ .safetensors, "consolidated.safetensors" }, - }; - - for (default_entrypoints) |entrypoint| { - const file_type, const file_name = entrypoint; - const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; - - return .{ file_type, file }; - } - - return error.FileNotFound; -} - -const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, -}); - -fn stringToDtype(safetensor_dtype: []const u8) !DataType { - return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); - return error.UnsupportedDataType; }; } + +test "resolveModelRepo refuses a file" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); + + // `tmpDir` is `.zig-cache/tmp/` under the cwd. + var path_buffer: [128]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/model.safetensors", .{&tmp.sub_path}); + try std.testing.expectError(error.InvalidPath, resolveModelRepo(io, path)); +} + +/// The entrypoint of the model directory `repo`, open, with its kind and +/// its name there: `model.safetensors.index.json`, else `model.safetensors`. +fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, EntrypointKind, []const u8 } { + // When neither opens, a denied probe is what is reported. On S3 it may + // mean absent: a bucket that may not be listed answers 403 for a + // missing key. + var denied = false; + const entrypoints = [_]struct { []const u8, EntrypointKind }{ + .{ "model.safetensors.index.json", .index }, + .{ "model.safetensors", .safetensors }, + }; + for (entrypoints) |entrypoint| { + const name, const kind = entrypoint; + if (repo.openFile(io, name, .{ .mode = .read_only })) |file| { + return .{ file, kind, name }; + } else |err| switch (err) { + error.Canceled => return ModelPathResolutionError.Canceled, + error.AccessDenied, error.PermissionDenied => denied = true, + else => {}, + } + } + return if (denied) ModelPathResolutionError.PermissionDenied else ModelPathResolutionError.FileNotFound; +} + +test "resolveModelEntrypoint falls back past a denied index and reports a denied probe" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const served = try tmp.dir.createDirPathOpen(io, "served", .{}); + defer served.close(io); + try served.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); + + // The testing io, but a missing file is `PermissionDenied`: S3 answers + // 403 for a missing key to a principal without `s3:ListBucket`. + var vtable = io.vtable.*; + vtable.dirOpenFile = struct { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + return std.testing.io.vtable.dirOpenFile(userdata, dir, sub_path, options) catch |err| switch (err) { + error.FileNotFound => error.PermissionDenied, + else => err, + }; + } + }.dirOpenFile; + const unlisted: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; + + const file, const kind, const name = try resolveModelEntrypoint(unlisted, served); + defer file.close(io); + try std.testing.expectEqual(.safetensors, kind); + try std.testing.expectEqualStrings("model.safetensors", name); + // `tmp.dir` holds neither entrypoint. + try std.testing.expectError(error.PermissionDenied, resolveModelEntrypoint(unlisted, tmp.dir)); + try std.testing.expectError(error.FileNotFound, resolveModelEntrypoint(io, tmp.dir)); +} + +fn stringToDtype(safetensor_type: []const u8) ?DataType { + const map = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type); +} diff --git a/zml/shape.zig b/zml/shape.zig index 386b594e..7f08992e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -117,31 +117,14 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) []const Shape.Tag { + pub fn parseTags(v: anytype) TagsArray { const T = @TypeOf(v); - if (T == []const Shape.Tag) return v; - comptime validateTags(T); - return comptime frozen: { - var tagz: [v.len]Shape.Tag = undefined; - for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; - // Force the creation of a comptime constant, so that we can return a pointer to it. - const frozen_tags = tagz; - break :frozen &frozen_tags; - }; - } - - fn validateTags(T: type) void { - const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); - switch (@typeInfo(T)) { - .@"struct" => |info| { - if (!info.is_tuple) @compileError(message); - for (info.field_types) |Field| { - if (Field != @EnumLiteral()) @compileError(message); - } - if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); - }, - else => @compileError(message), + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + var tags_: TagsArray = .empty; + inline for (v) |field| { + tags_.appendAssumeCapacity(toTag(field)); } + return tags_; } /// Create a shape from a struct literal, eg: @@ -424,6 +407,36 @@ pub const Shape = struct { return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); } + /// Compares dimensions, dtype, tags, mesh, and per-axis partition specs. + pub fn eqlWithTagsAndPartitioning(self: Shape, other: Shape) bool { + if (!self.eqlWithTags(other) or self._sharding.mesh != other._sharding.mesh) return false; + for (0..self.rank()) |ax| { + if (self.partition(ax) != other.partition(ax)) return false; + } + return true; + } + + test eqlWithTagsAndPartitioning { + const other_mesh: Sharding.Mesh = .initTest(.other, .{ .batch, .feature }); + const shape = Shape.init(.{ .out = 8, .in = 16 }, .f32) + .withPartitioning(&test_mesh, .{ .out = .feature, .in = .replicated }); + const Spec = Sharding.Partitioning.Logical.Spec; + // Only the first spec is `shape`'s. + const specs: []const Spec = &.{ .init(.feature), .init(.batch), .replicated, .open }; + for (specs, 0..) |spec, i| { + const other = shape.withPartitioning(&test_mesh, .{ .out = spec, .in = .replicated }); + try testing.expect(shape.eqlWithTags(other)); + try testing.expectEqual(i == 0, shape.eqlWithTagsAndPartitioning(other)); + } + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withPartitioning(&other_mesh, .{ .out = .feature, .in = .replicated }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withDtype(.bf16))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.setDim(.out, 9))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.rename(.{ .out = .features }))); + try testing.expect(!shape.eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8 }, .f32))); + try testing.expect(!shape.withDefaultPartitioning().eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8, .in = 16 }, .f32))); + try testing.expect(Shape.init(.{}, .f32).eqlWithTagsAndPartitioning(Shape.init(.{}, .f32))); + } + /// Format the shape. /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { @@ -825,18 +838,6 @@ pub const Shape = struct { } } - pub fn assertTags(self: Shape, tagz: anytype) void { - const T = @TypeOf(tagz); - if (T == Shape) { - return self.assertTags(tagz.tags()); - } else if (T != []const Shape.Tag) { - return self.assertTags(comptime parseTags(tagz)); - } - - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); - stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); - } - pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; @@ -856,12 +857,11 @@ pub const Shape = struct { .grid = undefined, }; - /// Returns shape metadata with the requested sharding on `mesh`. - /// Omitted dimensions in a mapping are replicated, including during compilation. - /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. + /// Partitions this shape on `mesh` as `Tensor.withPartitioning` takes `partitioning`, except + /// that axes it leaves unspecified are `.replicated`. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { - return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); + return self.withPartitioning(mesh, Sharding.Partitioning.Logical.init(partitioning).resolve(self.tags(), mesh, .replicated)); } var res = self; res._sharding = .{ .mesh = mesh, .partition = partitioning }; diff --git a/zml/tensor.zig b/zml/tensor.zig index ee084835..5dd62ddc 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -75,11 +75,14 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, - /// and open sharding for the partitioner to infer from graph constraints. + /// Internal use + /// + /// Creates a tensor from a Shape and an mlir.Value. + /// Note: sharding information is discarded, the output tensor is considered open for replication. + /// Use _resultPropagateSharding when the shape sharding should be propagated. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; - res._shape._sharding = .open; + res._shape._sharding.partition = .open(sh.rank()); if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. @@ -92,16 +95,16 @@ pub const Tensor = struct { return res; } - /// Internal: creates a constructor result and honors `sh`'s partitioning when - /// it specifies a mesh. pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { - const result = _result(sh, val); - const mesh = sh._sharding.mesh orelse return result; - return result.withPartitioning(mesh, sh._sharding.partition); + var res = _result(sh, val); + res._shape._sharding = sh._sharding; + return res; } - /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. - /// Dimension tags are unknown and sharding is open for partitioner inference. + /// Creates a Tensor from a mlir.Value + /// + /// The shape is derived from the type of the mlir.Value. + /// mesh is `unknown` pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -113,7 +116,7 @@ pub const Tensor = struct { ._dtype = ctx.dtype(ranked_tensor.elementType()), ._dims = .empty, ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, - ._sharding = .open, + ._sharding = .replicated, }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); @@ -163,22 +166,14 @@ pub const Tensor = struct { return res; } - /// Declares this tensor's sharding on `mesh_`. - /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. - /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension - /// tags to logical mesh axes, `.replicated`, or `.open`. + /// Specify the mesh of the input tensor. + /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get + /// a known mesh from the compilation options. + /// * partitioning: a `Partitioning`, or anything `Partitioning.Logical.init` accepts: a struct + /// mapping this tensor's axis tags to logical mesh axes, `.replicated` or `.open`. + /// Axes it leaves unspecified are `.open`. /// - /// During compilation, emits a graph constraint. For example, - /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` - /// and leaves omitted dimensions open for inference. Use `.replicated` to - /// keep a dimension whole on each shard. - /// - /// Outside compilation, updates the tensor's shape to describe an executable - /// input's sharding. Omitted dimensions are replicated and no graph constraint is - /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. - /// - /// Executable inputs and explicitly annotated outputs must have no open - /// dimensions. An output with no declared sharding defaults to replication. + /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { if (@TypeOf(mesh_) == @EnumLiteral()) { const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); @@ -186,8 +181,7 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); + const parsed = Sharding.Partitioning.Logical.init(partitioning).resolve(self._shape.tags(), mesh_, .open); return self.withPartitioning(mesh_, parsed); } @@ -230,14 +224,11 @@ pub const Tensor = struct { }, }; - var res = _result(partitioned_shape, op_result); - res._shape = partitioned_shape; - return res; + return _resultPropagateSharding(partitioned_shape, op_result); } - /// Emits a graph constraint requiring a complete copy on every device. - /// Requires active compilation. For a replicated input or buffer description, - /// use the default sharding from `Tensor.init` or `Shape.init`. + /// Force the input tensor to be fully replicated. + /// This can be detrimental for performance. pub fn replicate(self: Tensor) Tensor { const compiler = Compiler.current(); const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); @@ -278,17 +269,11 @@ pub const Tensor = struct { try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } - /// Copy the given tensor to the specified memory, preserving its sharding. + /// Copy the given tensor to the specified memory. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => { - // A single memory needs no copy, but the result must still be a distinct - // tensor: `reuseBuffer` donations are keyed by tensor id. - var res = _result(self._shape, self.value()); - res._shape._sharding = self._shape._sharding; - return res; - }, + .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } @@ -315,8 +300,7 @@ pub const Tensor = struct { ctx.location, ).appendTo(currentBlock()); - var res = _result(self._shape, op.result(0)); - res._shape._sharding = self._shape._sharding; + const res = _result(self._shape, op.result(0)); ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } @@ -863,7 +847,7 @@ pub const Tensor = struct { fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -904,8 +888,7 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - const result = floats.convert(shape_.dtype()); - return .{ rng, _resultPropagateSharding(shape_, result.value()) }; + return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; } test uniform { @@ -1534,7 +1517,7 @@ pub const Tensor = struct { pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { const T = @TypeOf(tags); if (comptime T == Shape.Tag or T == @EnumLiteral()) { - return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); + return lhs.dotAxes(rhs, .{tags}); } const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); @@ -2200,14 +2183,15 @@ pub const Tensor = struct { pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; + // `len` is checked before `end`, whose default would otherwise take the rest of the axis. const end = if (s.singleton) start + 1 + else if (s.len != to_the_end) + start + s.len else if (s.end == to_the_end) d else if (s.end < 0) d + s.end - else if (s.len != to_the_end) - start + s.len else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; @@ -2358,6 +2342,7 @@ pub const Tensor = struct { .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + .{ 1, Tensor.Slice{ .start = 1, .len = 2 }, [4]f32{ 1, 2, 6, 7 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; var exe = try zml.module.compile( @@ -2724,7 +2709,7 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), res_shape), currentLoc(), ).appendTo(currentBlock()); - return _resultPropagateSharding(res_shape, op.result(0)); + return _result(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -2799,8 +2784,7 @@ pub const Tensor = struct { } pub fn zeroes(sh: Shape) Tensor { - const result = Tensor.constant(sh.dtype().zero()).broad(sh); - return _resultPropagateSharding(sh, result.value()); + return Tensor.constant(sh.dtype().zero()).broad(sh); } pub fn uninitialized(sh: Shape) Tensor { @@ -2815,7 +2799,7 @@ pub const Tensor = struct { .appendTo(currentBlock()); const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); - return _resultPropagateSharding(sh, tensor.result(0)); + return _result(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. @@ -2823,7 +2807,7 @@ pub const Tensor = struct { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); - return _resultPropagateSharding(sh, constant_op.result(0)); + return _result(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2989,7 +2973,7 @@ pub const Tensor = struct { var res_shape = self._shape; const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; return self.reshape(res_shape); } @@ -4826,7 +4810,7 @@ pub const Tensor = struct { fn body(body_ctx: @This(), _: void) void { ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); } - }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); + }).body, .{ .input = input, .name = full_name }, {}); }, .oneapi, .neuron, .furiosa => {}, } diff --git a/zml/testing.zig b/zml/testing.zig index 5e3d6333..d1e6faf7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -57,8 +57,8 @@ fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) ! } else .{ slice_or_buffer, false }; } -/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the -/// host for comparison ! +/// Testing utility. Accepts both zml.Buffer and zml.Slice; a zml.Buffer is copied to the host for +/// comparison. pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; var left: Slice, const should_free_left = try toSlice(allocator, io, left_); @@ -365,20 +365,189 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } -fn BufferizedWithArgs(comptime T: type) type { - return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); -} +/// `device_count` devices for a loader test: the accelerator the tests run +/// on (`env`, shared and never freed) when it has that many, otherwise a CPU +/// platform with exactly that many, freed by `deinit`. `physical` lays the +/// first `device_count` devices on one axis, so a mesh over it (`mesh`) +/// places on exactly those, whatever the platform has. Must not move after +/// `init`: meshes point at `physical`. +pub const Devices = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + owned: ?*Platform, + physical: Sharding.PhysicalMesh, -fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { - var count: usize = 0; - var it = store.store.registry.iterator(); - while (it.next()) |entry| { - if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { - count += 1; - } + pub fn init(self: *Devices, allocator: std.mem.Allocator, io: std.Io, device_count: usize) !void { + const shared = env(); + const owned: ?*Platform = if (shared.target != .cpu and shared.devices.len >= device_count) + null + else + try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = @intCast(device_count) } }); + errdefer if (owned) |platform| platform.deinit(allocator, io); + const platform: *const Platform = if (owned) |p| p else shared; + const nodes = try allocator.alloc(Sharding.PhysicalNode, device_count); + defer allocator.free(nodes); + for (nodes, platform.devices[0..device_count]) |*node, device| node.* = .device(device); + self.* = .{ + .allocator = allocator, + .io = io, + .platform = platform, + .owned = owned, + .physical = try .fromTree(allocator, platform.target, .axis(.bus, .tree, nodes)), + }; } - return count; + pub fn deinit(self: *Devices) void { + self.physical.deinit(self.allocator); + if (self.owned) |platform| platform.deinit(self.allocator, self.io); + self.* = undefined; + } + + /// A mesh named `name` whose one axis, `model`, spans the devices. + pub fn mesh(self: *const Devices, name: []const u8) !Sharding.Mesh { + return .init(name, &self.physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); + } +}; + +pub const TensorBytes = struct { + name: []const u8, + /// The safetensors dtype name, such as "U16". + dtype: []const u8, + dims: []const i64, + bytes: []const u8, +}; + +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order, +/// and registers them from it. +pub fn writeSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + registry: *zml.safetensors.TensorRegistry, + tensors: []const TensorBytes, +) !void { + try writeSafetensorsFile(allocator, io, dir, sub_path, tensors); + const file = try dir.openFile(io, sub_path, .{}); + defer file.close(io); + try zml.safetensors.parseSafetensors(io, registry, file, ""); +} + +/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order. +pub fn writeSafetensorsFile( + allocator: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + sub_path: []const u8, + tensors: []const TensorBytes, +) !void { + var header: std.Io.Writer.Allocating = .init(allocator); + defer header.deinit(); + const writer = &header.writer; + try writer.writeByte('{'); + var offset: usize = 0; + for (tensors, 0..) |tensor, index| { + if (index != 0) try writer.writeByte(','); + try writer.print("\"{s}\":{{\"dtype\":\"{s}\",\"shape\":[", .{ tensor.name, tensor.dtype }); + for (tensor.dims, 0..) |dim, axis| try writer.print("{s}{d}", .{ if (axis != 0) "," else "", dim }); + try writer.print("],\"data_offsets\":[{d},{d}]}}", .{ offset, offset + tensor.bytes.len }); + offset += tensor.bytes.len; + } + try writer.writeByte('}'); + + const file = try dir.createFile(io, sub_path, .{}); + defer file.close(io); + var header_len: [8]u8 = undefined; + std.mem.writeInt(u64, &header_len, header.written().len, .little); + try file.writePositionalAll(io, &header_len, 0); + var position: u64 = header_len.len; + try file.writePositionalAll(io, header.written(), position); + position += header.written().len; + for (tensors) |tensor| { + try file.writePositionalAll(io, tensor.bytes, position); + position += tensor.bytes.len; + } +} + +/// Distinct, nonzero bytes for test tensor number `seed`. +pub fn pattern(bytes: []u8, seed: u8) void { + for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; +} + +/// A store over a copy of `registry`, for a test that builds several stores +/// over one checkpoint: a store takes its registry's entries. +pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.TensorStore.Options) !zml.io.TensorStore { + var copy: zml.safetensors.TensorRegistry = .init(allocator); + defer copy.deinit(); + for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); + return .init(allocator, ©, opts); +} + +/// Loads `model` from `store` with a loader of its own and returns its +/// buffers, which the caller frees with `mem.deinitBufferized`. A recipe +/// for tests: production code creates its loader before building the +/// model, so the loader's set-up runs meanwhile. +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + model: anytype, + store: *const zml.io.TensorStore, +) !zml.Bufferized(@TypeOf(model.*)) { + var buffers = try zml.mem.bufferize(allocator, model); + errdefer zml.mem.deinitBufferized(allocator, &buffers); + var loader = try zml.io.Loader.init(allocator, io, platform, .{}); + defer loader.deinit(); + try loader.start(model, &buffers, store, .none); + try loader.wait(); + return buffers; +} + +/// Reads every shard of `buffer` back, in its mesh's device order, and +/// checks that it lives in `memory`'s space and holds that device's box of +/// `global`, the whole tensor's row-major bytes (sub-byte elements packed). +/// A buffer without a mesh is replicated on the platform's devices. On +/// CPU/neuron/metal every memory kind is the one memory space, so only +/// CUDA/ROCm/oneAPI tell them apart. +pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.TensorStore.Memory) !void { + const platform = buffer._platform; + const shape = platform.placedShape(buffer.shape()).packedShape(); + try std.testing.expectEqual(shape.byteSize(), global.len); + const placement: Sharding.Placement = try .init(&shape); + const devices = placement.mesh().?.devicesInCanonicalOrder(); + try std.testing.expectEqual(devices.len, buffer._shards.len); + const shard_bytes = placement.localShape(shape).byteSize(); + const actual = try allocator.alloc(u8, shard_bytes); + defer allocator.free(actual); + const expected = try allocator.alloc(u8, shard_bytes); + defer allocator.free(expected); + const rank = shape.rank(); + const strides = shape.computeByteStrides(); + var shards = buffer.shards(); + for (devices, 0..) |device, device_index| { + const shard = shards.next().?; + try std.testing.expect(shard._pjrt_buffer.memory(platform.pjrt_api) == platform.devices[device.id].memory(memory.space()).?.pjrt_memory); + try shard.toHost(io, actual); + // The device's box, one run of the last axis per index of the others. + const box = placement.slices(device_index, shape); + const run: usize = if (rank == 0) shape.dtype().sizeOf() else @intCast(box.get(rank - 1).size * strides.get(rank - 1)); + var index: [zml.Shape.MAX_RANK]i64 = @splat(0); + var cursor: usize = 0; + while (cursor < expected.len) : (cursor += run) { + var offset: i64 = 0; + for (0..rank) |axis| offset += (box.get(axis).start + index[axis]) * strides.get(axis); + @memcpy(expected[cursor..][0..run], global[@intCast(offset)..][0..run]); + var axis = rank -| 1; + while (axis > 0) { + axis -= 1; + index[axis] += 1; + if (index[axis] < box.get(axis).size) break; + index[axis] = 0; + } + } + try std.testing.expectEqualSlices(u8, expected, actual); + } } /// Test a layer implementation against reference activations stored in a TensorStore. @@ -419,12 +588,12 @@ pub fn testLayer( const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, input_name); + const store_input = activation_store.withPrefix(input_name); + const expected_input_count = store_input.count(); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -442,7 +611,7 @@ pub fn testLayer( const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; const store_output = activation_store.withPrefix(output_name); - const expected_output_count = countWithPrefix(store_output, output_name); + const expected_output_count = store_output.count(); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -453,34 +622,27 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); - defer zml.meta.visit(struct { - fn cb(_: void, b: *zml.Buffer) void { - b.deinit(); - } - }.cb, {}, &args_buffers); + var args_buffers = try load(allocator, io, platform, &args, activation_store.store); + defer zml.mem.deinitBufferized(allocator, &args_buffers); exe_args.set(.{ layer_weights, args_buffers }); var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); - exe.call(io, exe_args, &exe_results, .{ .wait = true }); + try exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); defer for (results) |*result| result.deinit(); var failed: bool = false; - var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); - var reader = try store_output.getReader(subkey, io, &reader_buffer); - defer reader.deinit(); - const shape = store_output.getShape(subkey).?; - const expected_slice: zml.Slice = try .alloc(allocator, shape); + const tensor = store_output.get(subkey) orelse return error.TensorNotFound; + const expected_slice: zml.Slice = try .alloc(allocator, tensor.shape); errdefer expected_slice.free(allocator); - try reader.interface.readSliceAll(expected_slice.data()); + try tensor.read(io, expected_slice.data()); break :s expected_slice; }; defer expected_slice.free(allocator); @@ -508,6 +670,39 @@ pub fn testLayer( } } +test testLayer { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = env(); + + const Scale = struct { + weight: zml.Tensor, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + return x.mul(self.weight); + } + }; + const weight = [_]f32{ 2, -1, 0.5, 3 }; + const input = [_]f32{ 1, 2, 3, 4 }; + const output = [_]f32{ 2, -2, 1.5, 12 }; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ + .{ .name = "scale.weight", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&weight) }, + .{ .name = "x.in.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&input) }, + .{ .name = "x.out.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&output) }, + }); + var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); + defer store.deinit(); + + const layer: Scale = .{ .weight = store.view().withPrefix("scale").createReplicatedTensor("weight", null) }; + var layer_weights = try load(allocator, io, platform, &layer, &store); + defer zml.mem.deinitBufferized(allocator, &layer_weights); + try testLayer(allocator, io, platform, layer, .forward, store.view(), "x", layer_weights, &.{}, .{}); +} + fn center(slice: anytype, i: usize) @TypeOf(slice) { const c = 20; const start = if (i < c) 0 else i - c; @@ -518,3 +713,27 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { return std.testing.expectEqual(expected, actual); } + +test load { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var registry: zml.safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + var bytes: [8]u8 = undefined; + pattern(&bytes, 1); + try writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ + .{ .name = "weight", .dtype = "U8", .dims = &.{8}, .bytes = &bytes }, + }); + var store = try storeOf(allocator, ®istry, .{}); + defer store.deinit(); + // The store took a copy: the registry still holds its entries. + try std.testing.expectEqual(1, registry.tensors.count()); + + const Model = struct { weight: zml.Tensor }; + const model: Model = .{ .weight = store.view().createReplicatedTensor("weight", null) }; + var buffers = try load(allocator, io, env(), &model, &store); + defer zml.mem.deinitBufferized(allocator, &buffers); + try expectShards(allocator, io, &buffers.weight, &bytes, .default); +} diff --git a/zml/zml.zig b/zml/zml.zig index dafc3a0f..23d12e75 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); +pub const kernel = @import("kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; @@ -81,9 +81,3 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); -}