diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..529cdcf3 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,19 @@ +[[language]] +auto-format = true +file-types = [ "zig" ] +language-servers = [ "zls" ] +name = "zig" +roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] +scope = "source.zig" + +[language.indent] +tab-width = 4 +unit = " " + +[language-server.zls] +command = "sh" +args = [ + "-c", + "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", + "zls" +] diff --git a/README.md b/README.md index 216fc165..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,29 @@ const Mnist = struct { }; } + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + meshes: []const *const zml.Mesh, + ) !zml.Bufferized(Mnist) { + return zml.io.load(Mnist, self, allocator, io, platform, store, .{ + .meshes = meshes, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -172,46 +195,8 @@ const Mnist = struct { return x.argMax(0).indices.convert(.u8); } }; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const path = ...; - - // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - const io = vfs.io(); - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - // Returns at once: the loader calibrates while the model is compiled. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); - defer loader.deinit(); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); - defer store.deinit(); - const model: Mnist = .init(store.view()); - - const input: zml.Tensor = .init(.{ 28, 28 }, .u8); - var exe = try platform.compile(allocator, io, model, .forward, .{input}, .{}); - defer exe.deinit(); - - var weights = try zml.mem.bufferize(allocator, &model); - defer zml.mem.deinitBufferized(allocator, &weights); - try loader.start(&model, &weights, &store, .none); - try loader.wait(); // logs "Loaded weights" - - // ... call `exe` on the weights and an image -} ``` -The first `start` waits for whatever calibration is left. -[Loading checkpoints](./docs/learn/loader.md) gives the full recipe, with -remote checkpoints, placement, lifetimes and errors. - For a full walkthrough, see: - [Getting Started](./docs/tutorials/getting_started.md) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 434ecfa9..eb2b2ab2 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -3,7 +3,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "retain_runfiles_init", - # Mach-O adds an underscore to the exported Zig symbol. + srcs = ["runfiles_init.c"], + # Retain the constructor even when its object is in a static archive. + # Mach-O adds an underscore to the exported C symbol. linkopts = select({ "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 2ae3324a..67513227 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,35 +1,26 @@ const std = @import("std"); -const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; +pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; +} pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c new file mode 100644 index 00000000..770a4b91 --- /dev/null +++ b/bazel/runfiles_init.c @@ -0,0 +1,15 @@ +extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); + +// Preserve the existing startup ABI. Registration happens in C; +// runfiles discovery and initialization remain in Zig. +// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. +// The original Zig initializer already made this assumption to read argv[0]. +// +// Let C emit the platform's constructor registration. Zig's incremental ELF +// linker does not yet handle .init_array's section type, flags, and size correctly. +// TODO: Revert this workaround once Zig issue #37453 is fixed: +// https://codeberg.org/ziglang/zig/issues/37453 +__attribute__((constructor)) +void _bazel_runfiles_init(int argc, char **argv, char **envp) { + _bazel_runfiles_init_impl(argc, argv, envp); +} diff --git a/devenv.lock b/devenv.lock index bb3ef660..6c3be332 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1779636362, - "narHash": "sha256-Q/hy/ybvbe91fjA1MbulUwPjJH9yKaFZnSaTkwLE8S0=", + "lastModified": 1793199252, + "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", "owner": "cachix", "repo": "devenv", - "rev": "9d6e68dac81fab1f7f2ec6784220151b4fba4b59", + "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1778507786, - "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", + "lastModified": 1793043970, + "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "8f24a228a782e24576b155d1e39f0d914b380691", + "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1778274207, - "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", + "lastModified": 1792812153, + "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", + "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", "type": "github" }, "original": { diff --git a/docs/README.md b/docs/README.md index 3a356509..03369120 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,7 +24,6 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Learn more... - [ZML Concepts](./learn/concepts.md) : **Tensors, Models, Executables, etc. explained** -- [Loading checkpoints](./learn/loader.md) : **The loading recipe, placement, programs, lifetimes and errors** ## Contribute diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index d2c63bc1..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -160,6 +160,7 @@ bazel build //my_project:torch2zml ./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors +debug(zml/safetensors): Added new metadata key=format with value=pt info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` @@ -189,7 +190,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .init(allocator, &model_registry, .{}); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); defer model_store.deinit(); } ``` @@ -205,17 +206,17 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, .replicated, .{}), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), .d, ), .gate_proj = .init( mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, .replicated, .{}), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), .d, ), .down_proj = .init( mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), - mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, .replicated, .{}), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), .dout, ), }; @@ -234,16 +235,15 @@ Python script. Alternatively, Huggingface's web-interface allows to peek into ## Loading an individual layer Currently, the layer is initialized but the weights are still on the disk. -ZML provides a convenience to load weights from an initialized model in a test: +ZML provides a convenience to automatically load weights from an initialized model: ```zig pub fn main(init: std.process.Init) !void { ... var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - var mlp_weights = try zml.testing.load(allocator, io, platform, &mlp, &model_store); - defer zml.mem.deinitBufferized(allocator, &mlp_weights); + defer platform.deinit(allocator); + var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); } ``` @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .init(allocator, &activations_registry, .{}); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index 9f18b4a7..cc11ffbb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -17,7 +17,7 @@ access token to log into your account on Huggingface, directly from `bazel`, in * the token looks something like `hf_abCdEfGhijKlM`. * store the token on your machine (replace the placeholder with your actual token): -You can use the `HF_TOKEN` environment variable to store the token or use +You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use its standard location: ``` mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" @@ -29,6 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `hf auth login` command, or the `HF_TOKEN` environment variable. +# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/learn/loader.md b/docs/learn/loader.md deleted file mode 100644 index d97b363c..00000000 --- a/docs/learn/loader.md +++ /dev/null @@ -1,210 +0,0 @@ -# Loading checkpoints - -`zml.io.Loader` reads a model's weights from its checkpoint into device -buffers. Model code binds each tensor to checkpoint bytes through a -`TensorStore` view; the loader fills the buffers, assembles the tensors made -of several checkpoint tensors, and runs the programs that compute others. -The doc comments of `zml/io/loader.zig`, `TensorStore.zig`, `Source.zig` and -`vfs/vfs.zig` are the contract; the internals are in the `//!` header of -each file `zml/io.zig` lists. - -## The recipe - -```zig -// Local paths, and hf://, s3://, gs:// and https:// through `Remotes`, -// configured from the environment (tokens, credentials, proxies). -var vfs: zml.io.VFS = try .init(allocator, init.io); -defer vfs.deinit(); -var remotes: zml.io.VFS.Remotes = undefined; // initialized in place -try remotes.init(allocator, &vfs, init.environ_map); -defer remotes.deinit(); -const io = vfs.io(); // for the platform, the registry and the loader - -const platform: *zml.Platform = try .auto(allocator, io, .{}); -defer platform.deinit(allocator, io); - -// Returns at once: calibration runs while the model is built and compiled. -var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); -defer loader.deinit(); - -// The checkpoint's headers. -var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); -defer registry.deinit(); -var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); -defer store.deinit(); - -const model: Model = .init(store.view()); // binds every tensor -// ... compile the model ... - -var buffers = try zml.mem.bufferize(allocator, &model); -defer zml.mem.deinitBufferized(allocator, &buffers); -try loader.start(&model, &buffers, &store, .none); -try loader.wait(); // logs "Loaded weights [bytes, time, rate]" -``` - -`.checkpoint` lets the VFS backend serving the path size the reads, and the -loader place its pinned memory for how a local checkpoint reads; without it, -or without a VFS, reads are sized for local files. `start` adds its own items -to the progress node it is given. Several loads may be started before one -`wait`, from one store or several. A second checkpoint, such as a drafter's, -joins the registry under a prefix before the store takes it: -`registry.addRepo(io, drafter_repo, "drafter")`, then -`store.view().withPrefix("drafter")` reaches its tensors. - -## Placement - -A tensor is placed by its shape. `View.createTensor(subkey, tags, .tp, -partitioning)` places the checkpoint tensor on the mesh named `.tp` among the -store's `meshes` (`View.mesh(.tp)` returns it), each tag of `partitioning` -naming the mesh axis that splits it; `createReplicatedTensor` puts it whole -on every device. It lands in default device memory, unless `View.setMemory`, -called after binding, names a host kind (`TensorStore.Memory`): -`.host_pinned` gives buffers in pinned host memory, and `.host_mapped` gives -device buffers that are views of pinned host memory, which kernels read in -place. The second suits a table too large for device memory, of which each -step reads a few rows; the memory goes with the last view. XLA counts those -views against device memory: give their size on one device, -`view.hostMappedBytes()`, back through the compile option -`extra_device_memory_bytes`. - -```zig -const tp = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); -var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tp} }); -const view = store.view(); -const w = view.createTensor("w", .{ .out, .in }, .tp, .{ .out = .model }); -const b = view.createReplicatedTensor("b", .{.out}); -const rows = view.createReplicatedTensor("rows", .{ .row, .d }); -view.setMemory(rows, .host_mapped); -``` - -`View.get(subkey)` returns the registry's entry for a key, or null, and -`count()` the number of keys under the view's prefix. Binding cannot fail: -`createTensor` and `source` panic on a missing key, `maybeCreateTensor` and -`maybeSource` return null. `TensorStore.Options.quantization` -(`Quantization.Target.of(platform)`) is required to bind block-FP8 weights. - -## Composite tensors - -A composite tensor is one destination filled from several checkpoint byte -ranges, every byte no range covers set to zero, with no program and no -buffer per source. It is a `TensorStore.Source`: a checkpoint tensor -(`View.source(subkey, tags)`) and `zml.Tensor`'s shape ops over it, with the -same names, tags and arguments: `slice`, `split`, `chunkExact`, `splitAxis`, -`merge`, `reshape`, `transpose`, `pad` (with zeros), `broad`, `bitCast`, -`concatenate` and `stack`. Placement comes last, after every op: -`withPartitioning(mesh, partitioning)`, keyed by the source's final tags, or -`as(shape)`, a shape built elsewhere with the source's element count and -dtype. Both return a `Source.Placed`, which has no shape ops (its `.shape` -is the placed shape) and drops its placement with `unplaced()`; its -`bind()` makes the model tensor. - -```zig -const Source = zml.io.TensorStore.Source; -// a, b and c end to end along the rows, padded to 16 rows, row-sharded. -const abc = Source.concatenate(&.{ view.source("a", .{ .row, .d }), view.source("b", .{ .row, .d }), view.source("c", .{ .row, .d }) }, .row); -const table = abc.pad(0, .{ .row = Source.Pad{ .high = 16 - abc.dim(.row) } }) - .withPartitioning(.tp, .{ .row = .model }).bind(); -// q, k and v cut in one group per device and interleaved, so that each -// device holds its rows of every projection. -var members: [3]Source = undefined; -for (&members, [_][]const u8{ "q.weight", "k.weight", "v.weight" }) |*m, key| { - m.* = view.source(key, .{ .dout, .d }).splitAxis(.dout, .{ .group = devices, .r = -1 }); -} -const qkv = Source.concatenate(&members, .r).merge(.{ .dout = .{ .group, .r } }).withPartitioning(.tp, .{ .dout = .model }).bind(); -``` - -Sources live in their store's arena, and sources of two stores do not mix. -The loader only moves bytes: large runs are cheap, runs under 4 KiB cost -host copies. For a layout of many small runs, a program over sources read -in large runs may load faster, at the cost of a compile and staging memory -(measured in the header of `zml/io/direct_loader.zig`). - -## Programs - -What computes rather than moves bytes is a program: a dtype conversion, a -dequantization, a global scale. A program is a value of its own type, plain -data (checked at compile time), with a `forward` method; its type name -names the compiled module in traces and logs. - -```zig -const Dequantize = struct { - block_size: i64, - - pub fn forward(self: Dequantize, output: zml.Shape, inputs: []const zml.Tensor) zml.Tensor { - // One scale per row and block of `block_size` columns. - const blocks = inputs[0].splitAxis(.in, .{ .block = -1, .k = self.block_size }); - const y = blocks.mul(inputs[1].broad(blocks.shape())).merge(.{ .in = .{ .block, .k } }); - return y.convert(output.dtype()).withPartitioning(output._sharding.mesh.?, output._sharding.partition); - } -}; -const w = view.source("w", .{ .out, .in }); -const s = view.source("w_scale", .{ .out, .block }); -// Each device reads its rows of `w`, and all of `s`. -const placed = w.shape().withPartitioning(tp, .{ .out = .model }); -const weight = Source.bindProgram(&.{ w.as(placed), s.withPartitioning(.replicated, .{}) }, placed.withDtype(.bf16), Dequantize{ .block_size = 128 }); -``` - -`bindProgram(inputs, output, program)` binds a tensor of `output` that -`program.forward(output, inputs)` computes during the load and returns -placed as `output` is. Each input is a `Source.Placed` and lands placed by -its own shape, so every device reads only its slice; a whole checkpoint -tensor, replicated, is -`view.source(key, null).withPartitioning(.replicated, .{})`. `start` -starts one compile for all the tensors of an equal program (same type, -equal value) over the same placed shapes. `wait` lands the inputs in groups -of at most `Loader.Options.program_group_bytes` of outputs (8 GiB), runs -the programs on the calling task and frees the inputs. While other groups -are pending, a group waits until the devices have room for it; once none -is pending, it goes anyway, and a group that does not fit a device fails -`wait` with the device's error. Before `wait` returns, the load gives the -device memory it freed, inputs and temporaries, back to the devices; XLA's -VMM allocator, the default on CUDA and ROCm, would otherwise keep counting -it as in use. A program's output lands in default device memory: -`setMemory` panics on it. - -## Lifetime and failure - -| Handle | Borrows | Until | -| --- | --- | --- | -| `Loader` | the platform; `io`, and the VFS behind it, through which it may keep its source files open | its `deinit` | -| `Loader.start` | the store, the buffers and the progress node | `wait` or `cancel` returns | -| `TensorStore` | the meshes; it owns the registry's entries, the bindings, sources and programs | its `deinit` | -| `View`, `Source`, `Source.Placed` | their store, which must not move | their last use | -| `VFS.Remotes` | the VFS; it must not move | its `deinit`, once every file opened through it is closed | -| Loaded buffers | the platform | `deinitBufferized` | - -`init` reads its options and `start` the model during the call only. The -loader copies each file's path, so nothing of the store is kept past `wait`. -The loaded buffers are the caller's and outlive the loader, which holds its -pinned memory and workers until `deinit`: the recipe's deferred `deinit` -keeps them until the function returns, which suits a program that does -little after the load; one that keeps running frees the loader right after -`wait`, or loads in a block of its own. Deferred cleanup in the recipe's -order is right; if code between `start` and `wait` can fail, -`errdefer loader.cancel();` after `start`. The loader is a value. Start and -wait on one task: the backend reads and transfers concurrently. - -Every error of `start` or `wait` fails the loader: everything started is -canceled, and every later call returns the first error. `cancel` gives up on -what is started without running its programs. -Whatever failed, each buffer is empty or holds shards whose contents are -unspecified: read none, and free the tree with `zml.mem.deinitBufferized`. -That holds for the buffers `bufferize` made, which `start` empties again. - -## Errors - -`Loader.init` returns `InvalidOptions`, logged with the option it refuses, -or `OutOfMemory` when it cannot copy a local checkpoint's path; an error of -the backend's set-up is returned by the first `start`. -`start` and `wait` return the error where it was detected, under its own -name, and that is where it is logged, once, at err, with what explains it; -a plain allocation failure and `Canceled` are returned unlogged. - -A model whose bindings are wrong is `InvalidBinding`; the log names the -tensor (for a program's output, the program), the key its source reads and -the rule it breaks (`Loader.start`, `Loader.wait`). - -The other errors a caller can act on are listed in `Loader`'s doc comment: -a missing or denied file, a file that ends inside a tensor, host or device -memory that runs out, an `io` without concurrency, and `Canceled`. PJRT, -compile and remote failures pass through as their source names them. diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index fc394979..312c9afc 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -6,8 +6,7 @@ - clone ZML to work directly within the prepared example folder - add Zig code to implement our model - add some Bazel to integrate our code with ZML -- no weights files or anything external is required, until the last step - loads the weights from a checkpoint +- no weights files or anything external is required for this example The reason we're doing our exercise in the `examples` folder is because it's especially prepared for new ZML projects. It contains everything needed for ZML @@ -59,12 +58,10 @@ _Module_. For compilation, only the _Tensors_ are required. No actual tensor dat is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ holds the shapes -of a checkpoint's tensors, so that we can make _Tensors_, and the _Loader_ later -materializes them to _Buffers_. At first we won't use them: we will directly -create tensors using their shape and data type, and fill their buffers from -slices. The last step, [Loading weights](#loading-weights), reads them from a -checkpoint instead. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything +that is required to make _Tensors_ and later materialize them to _Buffers_. In our +example though, we won't even use a _TensorStore_ as it's optional. We will directly +create tensors using their shape and data type. After compilation is done, we get what we call an _Executable_. From this _Executable_ we can create _Args_ and _Results_, two structs that will store respectively the inputs @@ -178,9 +175,9 @@ var executable = try platform.compile(allocator, io, layer, .forward, .{input}, defer executable.deinit(); ``` -ZML supports sharding tensors across multiple devices, and `compile()` reads how -each tensor is sharded from its shape. A tensor created from a shape alone, as -here, is replicated on every device. +You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, +and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. +For this simple example, we just replicate the tensors across all devices for simplicity. ### Creating the "bufferized" Model @@ -194,8 +191,8 @@ provides ways to have extra-low loading times). const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); @@ -216,7 +213,7 @@ To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); // create the Args and Results structs @@ -230,7 +227,7 @@ defer results.deinit(allocator); args.set(.{ layer_buffers, input_buffer }); // call our executable -try executable.call(io, args, &results, .{}); +executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -375,15 +372,15 @@ pub fn main(init: std.process.Init) !void { const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); // create the Args and Results structs @@ -397,7 +394,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ layer_buffers, input_buffer }); // call our executable - try executable.call(io, args, &results, .{}); + executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -414,168 +411,10 @@ pub fn main(init: std.process.Init) !void { } ``` -## Loading weights - -Real models read their weights from a checkpoint. Let's write one for our -layer, a `.safetensors` file holding `weight` and `bias`, with a few lines of -Python: - -```python -import json, struct - -weight = struct.pack("<3e", 1.0, 2.0, 3.0) # three f16 values -bias = struct.pack("<3e", 1.0, 1.0, 1.0) -header = json.dumps({ - "weight": {"dtype": "F16", "shape": [3], "data_offsets": [0, 6]}, - "bias": {"dtype": "F16", "shape": [3], "data_offsets": [6, 12]}, -}).encode() -with open("/tmp/layer.safetensors", "wb") as f: - f.write(struct.pack(" .cta_1, + .two => .cta_2, + }; + } +}; +pub const MBarrierScope = nvvm.MBarrierScope; +pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; + +/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. +pub fn ctaCount(group: CtaGroup) u8 { + return @backingInt(group); +} + +pub const Tcgen05SmemSwizzle = enum(u3) { + none = 0, + swizzle_128b_atom_32b = 1, + swizzle_128b = 2, + swizzle_64b = 4, + swizzle_32b = 6, +}; + +/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. +pub const Tcgen05SmemDesc = struct { + pub const Options = struct { + leading_byte_offset: u32 = 0, + stride_byte_offset: u32 = 0, + version: u2 = 1, + base_offset: u3 = 0, + layout: Tcgen05SmemSwizzle = .none, + }; + + pub fn build(b: *Builder, start_address: Value, opts: Options) Value { + std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); + std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); + const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | + (@as(u64, opts.stride_byte_offset >> 4) << 32) | + (@as(u64, opts.version) << 46) | + (@as(u64, opts.base_offset) << 49) | + (@as(u64, @backingInt(opts.layout)) << 61); + const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); + } +}; + +/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. +pub const Tcgen05InstrDesc = struct { + pub const Options = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType = .f32, + m_dim: u16, + n_dim: u16, + a_major: OperandMajorMode = .k, + b_major: OperandMajorMode = .k, + a_negate: bool = false, + b_negate: bool = false, + }; + + pub fn build(comptime opts: Options) u32 { + comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); + comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); + const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); + const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); + const valid = (half_inputs and opts.c_dtype == .f32) or + (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or + (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or + (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or + (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); + if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); + const c: u32 = switch (opts.c_dtype) { + .f16 => 0, + .f32 => 1, + .i32 => 2, + else => @compileError("unsupported tcgen05 accumulator dtype"), + }; + return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | + (@as(u32, @intFromBool(opts.a_negate)) << 13) | + (@as(u32, @intFromBool(opts.b_negate)) << 14) | + (@as(u32, @backingInt(opts.a_major)) << 15) | + (@as(u32, @backingInt(opts.b_major)) << 16) | + (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); + } + + fn inputFormat(comptime dtype: DType) u32 { + return switch (dtype) { + .f16, .f8e4m3fn => 0, + .bf16, .f8e5m2, .i8 => 1, + .f32 => 2, + .f4e2m1fn => 5, + else => @compileError("unsupported tcgen05 input dtype"), + }; + } +}; + +/// Typed description of a non-multicast tiled TMA atom: the element type, +/// bits moved per copy, basis of one box, and coordinate layout of the tensor. pub const TmaConfig = struct { dtype: DType, copy_bits: u32, @@ -475,18 +634,29 @@ pub const TmaConfig = struct { coordinate_layout: LayoutSpec, coordinate_rank: u8, format: TmaFormat = .default, + /// Load completion group: with `.two`, each CTA's load completes on + /// the mbarrier of the pair's even CTA. Ignored by TMA stores. + cta_group: CtaGroup = .one, + mode: enum { tiled, gather4 } = .tiled, }; +pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensorTileS2GOp = struct {}; + pub const BlockScaledMma = struct { atom: Atom, tiled: Atom, }; -/// SM100 block-scaled MMA instruction: one CTA computes a 128 x `n` tile, -/// FP4 weights as A. The instruction `kind` fixes B, the scales and K. +/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. +/// The instruction `kind` fixes B, the scales and K. pub const BlockScaledMmaConfig = struct { kind: Kind = .mxfp4_mxfp8, + /// M is 128 for one CTA, or 128 or 256 for a CTA pair. + m: u16 = 128, n: u16, + cta_group: CtaGroup = .one, pub const Kind = enum { /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 @@ -524,8 +694,23 @@ pub const BlockScaledMmaConfig = struct { .nvfp4_nvfp4 => 16, }; } + + fn validate(self: BlockScaledMmaConfig) void { + const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); + const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; + if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); + } + } }; +fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { + return switch (config.cta_group) { + .one => .sm_90, + .two => .sm_100_2sm, + }; +} + fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { .u4 => .U4, @@ -573,6 +758,7 @@ pub const ArgSpec = struct { pub const MlirTypeSpec = struct { type_: *const mlir.Type, grid_constant: bool = false, + cuda_grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -681,6 +867,7 @@ pub const Builder = struct { .mlir_type => .{ .mlir_type = .{ .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, } }, } }; } @@ -707,7 +894,7 @@ pub const Builder = struct { return @Struct(.auto, null, &names, &types, &@splat(.{})); } - fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { std.debug.assert(self.entry_block == null); const ctx = self.ctx; const scratch = self.arena.allocator(); @@ -729,7 +916,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -941,6 +1128,75 @@ pub const Builder = struct { })); } + /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. + pub fn tensorMapType(self: *Builder) *const mlir.Type { + return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); + } + + /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / + /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. + fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { + const ctx = self.ctx; + const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); + const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); + const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, + .location = self.loc(), + })); + for (values, 0..) |v, i| { + const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = &.{array.inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, + .location = self.loc(), + })); + const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); + self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); + } + return array; + } + + /// Encode a tiled map in a host function and return both the map and CUDA status. + /// The caller supplies the final device address and handles the returned status. + pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { + const ctx = self.ctx; + // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. + ctx.setAllowUnregisteredDialects(true); + const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); + const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( + ctx, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, + self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, + addr.inner, + self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, + self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, + cuda.resultType(ctx), + self.tensorMapType(), + self.loc(), + )); + return .{ .map = results[1], .result = results[0] }; + } + + /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` + /// (`&tensor_map` for a `__grid_constant__` parameter). + pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { + const ctx = self.ctx; + const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); + return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{llvm_ptr.inner} }, + .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, + .location = self.loc(), + })); + } + pub fn cudaResultStatus(self: *Builder, result: Value) Value { return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); } @@ -1019,7 +1275,8 @@ pub const Builder = struct { /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared /// memory, `i64` for global memory), for hand-written PTX operands. pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - return self.emit(cute_ir.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); + const ptr = self.asValue(pointer); + return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); } /// Reinterpret a pointer without changing its address. This is the typed @@ -1074,6 +1331,7 @@ pub const Builder = struct { const result = switch (e) { .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), .basis => |b| if (@hasDecl(A, "getBasis")) blk: { const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); @@ -1238,8 +1496,59 @@ pub const Builder = struct { // ==================== cute.core ==================== - /// `cute.make_layout(shape, stride=...)`, static only. - pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + fn layoutResult(comptime Shape: type) type { + return switch (@typeInfo(Shape)) { + .pointer => Layout, + else => View, + }; + } + + /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static + /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. + pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { + if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ + .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, + }); + var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const shape_expr = self.coordExpr(&shape_values, shape); + const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { + var stride: AlgebraExpr = .{ .static = 1 }; + var stride_value: ?Value = null; + var index: usize = 0; + break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); + }; + const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); + const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); + const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); + const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { + if (shape == .tuple) { + const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); + for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); + return .{ .tuple = elements }; + } + if (shape == .static and shape.static == 1) return .{ .static = 0 }; + const result = stride.*; + if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); + if (shape == .static and stride.* == .static) { + stride.static *= shape.static; + } else { + const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { + const v: Value = .{ .inner = values[index.*], .kernel = self }; + index.* += 1; + break :blk v; + } else self.lift(shape.static); + stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); + stride.* = dynamicExpr(stride_value.*.?); + } + return result; + } + + fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); const stride = opts.stride orelse self.leftMost(shape); if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); @@ -1262,8 +1571,8 @@ pub const Builder = struct { fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { const T = @TypeOf(coord); if (T == Value) { - dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); - return .{ .dynamic = 1 }; + dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); + return dynamicExpr(coord); } if (T == AlgebraToken) return switch (coord) { .all => .all, @@ -1282,6 +1591,12 @@ pub const Builder = struct { } } + fn dynamicExpr(value: Value) AlgebraExpr { + std.debug.assert(value.isInt()); + const width = dtypeBitwidth(value.dtype()); + return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; + } + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` /// placeholders. Integer leaves stay static in the dependent CuTe type. pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { @@ -1516,20 +1831,27 @@ pub const Builder = struct { /// `projection`, like Python's `proj`, selects the modes of `tile` that /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { + if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, - result_type, + result_type.?, .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } + // A null result type on the algebra operations below requests dialect inference. + fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { + const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); + return .{ .inner = value.inner, .kernel = self }; + } + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.local_partition( self.ctx, @@ -1542,12 +1864,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1566,13 +1889,17 @@ pub const Builder = struct { (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() else - std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); + std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| memref.ptr_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + desc.desc() else unreachable; const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); @@ -1580,11 +1907,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1603,12 +1931,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1625,39 +1954,44 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1694,44 +2028,94 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { - self.emitVoid(cute_ir.copy(self.ctx, atom.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (pred) |p| p.inner else null }, self.loc())); + pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { + var exec = atom; + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA store atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + if (opts.tma_bar_ptr) |mbar| { + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .gather4, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable gather atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + exec = self.setTmaBarrier(exec, mbar); + } + if (opts.cache_policy) |policy| { + const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) + (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else + @panic("cache_policy requires an executable TMA copy atom"); + const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); + exec = .{ .inner = updated.inner, .kernel = self }; + } + self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { @@ -1762,9 +2146,67 @@ pub const Builder = struct { // the kernel receives them as grid-constant arguments and makes them // executable before issuing copies. + pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; + + fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { + return self.emit(mlir.Operation.make(self.ctx, name, .{ + .operands = .{ .flat = operands }, + .attributes = attributes, + .result_type_inference = true, + .location = self.loc(), + })); + } + + fn valueShape(self: *Builder, value: Value) Value { + if (value.type_().isA(cute_ir.ShapeType) != null) return value; + return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); + } + + fn shapeRank(shape: Value) usize { + return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); + } + + /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout + /// at construction and passes data/index tensors together to partition/copy. + pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { + const Tiler = @TypeOf(cta_tiler); + const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) + self.asValue(cta_tiler) + else if (@typeInfo(Tiler) == .pointer) + self.makeShape(self.dimsExpr(cta_tiler)).value() + else + self.makeShape(cta_tiler).value(); + var smem = self.asValue(smem_layout); + const tiler_rank = shapeRank(self.valueShape(tiler)); + const smem_rank = shapeRank(self.valueShape(smem)); + std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); + if (smem_rank == tiler_rank + 1) { + const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); + for (modes, 0..) |*mode, i| mode.* = @intCast(i); + smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); + } + const shape = self.valueShape(self.asValue(gmem)); + const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); + const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); + var cfg = config; + cfg.mode = .tiled; + if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); + cfg.cta_group = op.cta_group; + if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); + if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); + cfg.mode = .gather4; + const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); + const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); + const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); + const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); + return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; + } + /// Build the host-visible TMA descriptor and its coordinate tensor, whose /// types are assembled from `config`. pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + std.debug.assert(config.mode == .tiled); const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( self.ctx, self.asValue(gmem).inner, @@ -1772,7 +2214,7 @@ pub const Builder = struct { self.asValue(cta_map).inner, self.tmaLoadAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = .sm_90 }) catch @panic("invalid tiled TMA load")).attribute(), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); @@ -1802,8 +2244,15 @@ pub const Builder = struct { /// Host-side tiled TMA load atom type described by `config`. pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ + .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid gather TMA atom")).type_(); return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ - .kind = .sm_90, + .kind = tmaLoadKind(config), .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), .tmaGbasis = self.layoutType(config.global_basis), @@ -1818,8 +2267,8 @@ pub const Builder = struct { return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), - .mode = .tiled, - .numCta = 1, + .mode = if (config.mode == .gather4) .gather4 else .tiled, + .numCta = ctaCount(config.cta_group), .gStride = self.emptyStrideType(), .tmaGbasis = self.layoutType(config.global_basis), }) catch @panic("invalid TMA load atom")).type_(); @@ -1851,6 +2300,12 @@ pub const Builder = struct { return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); } + /// Generic address (i64) of the tensor map in an executable tiled TMA atom. + pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { + const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); + return self.ptrToInt(ptr, .i64); + } + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); } @@ -1861,11 +2316,11 @@ pub const Builder = struct { } /// Attach the mbarrier that an executable TMA load atom completes. - pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { + pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, atom.inner, - barrier.inner, + mbar.inner, atom.type_(), (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), self.loc(), @@ -1915,10 +2370,10 @@ pub const Builder = struct { } fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - if (cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) std.debug.panic("unsupported block-scaled MMA N {d}", .{cfg.n}); + cfg.validate(); return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ 128, cfg.n, cfg.k() })).attribute(), - .numCta = 1, + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), + .numCta = ctaCount(cfg.cta_group), .aMajor = .k, .bMajor = .k, .aType = DType.f4e2m1fn.toMlir(self.ctx), @@ -1952,10 +2407,27 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected - /// lane of the MMA warp issues this after each K-stage. - pub fn tcgen05Commit(self: *Builder, barrier: Value) void { - self.emitVoid(nvvm.tcgen05_commit(self.ctx, self.sharedBarrierPtr(barrier).inner, .cta_1, self.loc())); + pub const Tcgen05CommitOptions = struct { + /// Must match the group of the operations being tracked. + cta_group: CtaGroup = .one, + /// Arrive at the same barrier offset in each CTA whose rank bit is set. + /// Without a mask, arrive only on the issuing CTA's barrier. + multicast_mask: ?Value = null, + }; + + /// Track this thread's preceding tcgen05 MMA/copy operations of the + /// selected group, signaling one arrival per destination on completion. + /// The same elected thread that issues those operations must commit them. + /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 + /// operations in a kernel must use the same group. + pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { + self.emitVoid(nvvm.tcgen05_commit( + self.ctx, + self.sharedBarrierPtr(mbar).inner, + opts.cta_group.toNvvm(), + if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, + self.loc(), + )); } /// Wait until asynchronous tensor-memory loads have reached registers. @@ -1963,14 +2435,14 @@ pub const Builder = struct { self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); } - pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; + pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, self.constant(.i32, columns).inner, self.asValue(holding_ptr).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -1981,7 +2453,7 @@ pub const Builder = struct { } pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null }, self.loc())); + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); } pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { @@ -1989,7 +2461,7 @@ pub const Builder = struct { self.ctx, self.asValue(pointer).inner, self.constant(.i32, columns).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2182,22 +2654,37 @@ pub const Builder = struct { _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); } - /// Synchronize a fixed subset of the CTA on a named hardware barrier. - /// `participants` must be a warp multiple and every participating thread - /// must execute the same barrier id. - pub fn namedBarrier(self: *Builder, id: i32, participants: i32) void { - // The embedded CuTe compiler currently reads NVVM through a different - // MLIR revision. A generic `nvvm.barrier` with optional operands is - // accepted, but those operands are discarded and it lowers to - // `bar.sync 0`. Constant inline PTX preserves the named barrier ABI - // used by CUTLASS (`2,160` for TMEM allocation and `1,128` for the - // epilogue pipeline). - const assembly = std.fmt.allocPrint( - self.arena.allocator(), - "bar.sync {d}, {d};", - .{ id, participants }, - ) catch @panic("namedBarrier OOM"); - _ = llvm.inline_asm(self.ctx, assembly, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). + /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be + /// a positive warp multiple, and every participating thread must execute + /// the same barrier id. The aligned form requires CTA-uniform control flow. + pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { + if (@TypeOf(participants) == ?Value) { + if (participants) |count| return self.barrierCtaSyncAligned(id, count); + return self.barrierCtaSyncAligned(id, null); + } + if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { + std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); + } + if (@TypeOf(id) == Value) { + std.debug.assert(id.dtype() == .i32); + } else { + const immediate: i32 = id; + std.debug.assert(immediate >= 0 and immediate < 16); + } + // The LLVM intrinsic preserves convergence and both operands across + // the embedded compiler's NVVM dialect version boundary. + if (@TypeOf(participants) == @TypeOf(null)) { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); + } + } + + pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; + + pub fn barrier(self: *Builder, opts: BarrierOptions) void { + self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2368,48 +2855,101 @@ pub const Builder = struct { /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe /// shared-memory pointer or a raw `i32` shared-memory address. - fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { - if (barrier.isInt()) return self.addressPtr(barrier, .shared); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{barrier.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { + if (mbar.isInt()) return self.addressPtr(mbar, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. - pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, arrivals).inner, self.loc())); + pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); } - pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { - self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, count).inner, .cta, self.loc())); + pub const MBarrierArriveOptions = struct { + peer_cta_rank_in_cluster: ?Value = null, + arrive_count: ?Value = null, + scope: MBarrierScope = .cta, + relaxed: bool = false, + }; + + /// Arrive locally or at the same offset in a peer CTA. Local arrivals + /// return the opaque state token; remote arrivals have no state result. + pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { + var ptr = self.sharedBarrierPtr(mbar); + if (opts.peer_cta_rank_in_cluster) |rank| { + ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ + .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, + .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { + .cta => .cta, + .cluster => .cluster, + } }) catch unreachable).attribute(), + .relaxed = .boolean(self.ctx, opts.relaxed), + .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, + }, self.loc()); + if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); + self.emitVoid(op); + return null; } - /// Test whether phase `phase` has completed, without suspending (SM90+). - pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, .@"try", .cta, self.loc())); + pub const MBarrierTryWaitOptions = struct { + scope: MBarrierScope = .cta, + time_limit: ?Value = null, + }; + + /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend + /// briefly and return false before completion. `scope` selects which + /// threads this wait can synchronize with; the barrier remains local. + pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { + const ptr = self.sharedBarrierPtr(mbar); + if (opts.time_limit) |ticks| { + return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); + } + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); } - /// Wait for phase `phase`, letting the warp sleep for up to - /// `suspend_time` cycles per attempt (SM90+). - pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { - self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.constant(.i32, suspend_time).inner, self.loc())); + pub const MBarrierWaitOptions = struct { + scope: MBarrierScope = .cta, + /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. + time_limit: u32 = 10_000_000, + /// Probe before suspending, as in CuTe's pipeline consumer wait. + test_first: bool = true, + }; + + fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + const ptr = self.sharedBarrierPtr(mbar); + const ticks = self.constant(.i32, opts.time_limit); + switch (opts.scope) { + // Preserve the operation recognized by CuTe's pipeline compiler. + .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), + .cluster => { + var pending = self.openWhile(.{}, .{}); + const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); + pending.yieldBefore(ready.eq(false), .{}); + pending.yieldAfter(.{}); + }, + } } - /// Wait for a pipeline phase with the two-step sequence of CuTe's - /// `PipelineTmaUmma`: test the barrier without suspending, then suspend - /// only when the phase is still outstanding. The initial test matters when + /// Acquire a pipeline phase at `scope`: test without suspending, then + /// block only when the phase is still outstanding. The initial test matters when /// a producer reaches a full ring and must yield to the consumer that /// recycles its next stage. - pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { - const ready = self.mbarrierWaitParity(barrier, phase); - var pending = self.openIf(ready.eq(false)); - { - self.mbarrierTryWaitParity(barrier, phase, 10_000_000); + pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + if (opts.test_first) { + const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); + var pending = self.openIf(ready.eq(false)); + self.mbarrierWaitLoop(mbar, phase, opts); pending.close(); + } else { + self.mbarrierWaitLoop(mbar, phase, opts); } } @@ -2438,7 +2978,7 @@ pub const Builder = struct { // Hand-scheduled kernels address shared memory with `i32` offsets in the // shared window, global memory with `i64` addresses and tensor memory with // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / - // LLVM operations for them directly (no inline PTX). + // LLVM operations for them directly, using inline PTX where needed. pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; @@ -2500,11 +3040,6 @@ pub const Builder = struct { self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); } - /// `bar.sync id, threads`: named barrier over `threads` threads. - pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads) }, null); - } - /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether /// `predicate` (i1) is true in any of them. pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { @@ -2516,9 +3051,10 @@ pub const Builder = struct { self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); } - /// Non-blocking test of mbarrier phase `parity` (i1). - pub fn mbarrierTestParity(self: *Builder, barrier: Value, parity: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, parity.inner, .@"test", .cta, self.loc())); + /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). + pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { + std.debug.assert(opts.time_limit == null); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); } /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of @@ -2549,8 +3085,8 @@ pub const Builder = struct { /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with /// `noinc`, the arrival is counted in the barrier's expected count. - pub fn cpAsyncMbarrierArrive(self: *Builder, barrier: Value, comptime noinc: bool) void { - _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(barrier)}, null); + pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); } /// Wait until all of this thread's cp.async copies have landed. @@ -2705,13 +3241,271 @@ pub const Builder = struct { return self.gridDim().z.mul(value); } + /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). + /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), + /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). + /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` + /// are element coordinates; row length and out-of-bounds fill come from the + /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. + pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { + std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); + for (r) |row| std.debug.assert(row.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; + + /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. + pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { + if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); + const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); + self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ + .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, + }, self.loc())); + } + + pub fn prefetchL1(self: *Builder, addr: Value) void { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn prefetchTensorMap(self: *Builder, desc: Value) void { + const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const OperandSource = union(enum) { smem: Value, tmem: Value }; + pub const Tcgen05MmaWsOptions = struct { + collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, + collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, + col_b_zero_mask: ?Value = null, + }; + + /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. + /// The caller elects the issuer and commits/waits before consuming D. + pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { + std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); + const adesc = switch (a) { + .smem => |desc| self.smemDescriptorToInt(desc), + .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), + }; + const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); + self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ + .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, + }, self.loc())); + } + + pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { + return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { + .k => .k, + .mn => .mn, + } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); + } + + pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); + } + + pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { + if (desc.isInt()) return desc.to(.i64); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); + } + + pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .increase); + } + + pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .decrease); + } + + fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { + comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); + const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); + _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. + pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { + std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, switch (scope) { + .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. + fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { + if (!ptr.isPtr()) return ptr; + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } + + /// Map a local shared pointer/address to a typed pointer in peer CTA memory. + pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { + const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); + const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem); + const result_type = (cute_ir.PtrType.get(self.ctx, .{ + .valueType = ty.valueType(), + .addressSpace = .dsmem, + .alignment = ty.alignment(), + .swizzle = ty.swizzle(), + .bitlayout = ty.bitlayout(), + }) catch @panic("invalid DSMEM pointer")).type_(); + return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); + } + return remote; + } + + /// Store one, two, or four i32 words in a peer CTA, completing bytes on its + /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. + pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { + const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); + } + if (mbar.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); + } + const packed_value = if (@TypeOf(value) == Value) value else blk: { + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + break :blk self.joinWords(n, words); + }; + self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); + } + + /// Explicit aligned variants retain the convergence requirement of raw PTX. + pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrierClusterWaitAligned(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + pub const ClusterArriveOptions = struct { aligned: bool = false }; + + pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { + if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn clusterWait(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); + } + + /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. + pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// A null mask selects CuTe's FULL_MASK. + pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; + } + + pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; + + pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { + if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; + return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ + .abs = if (opts.abs) .unit(self.ctx) else null, + .nan = if (opts.nan) .unit(self.ctx) else null, + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }, self.loc())); + } + + fn vec2(self: *Builder, x: [2]Value) Value { + const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); + return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); + } + + fn unvec2(self: *Builder, v: Value) [2]Value { + return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; + } + + pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; + + fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { + return .{ + .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }; + } + + pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub const StoreOptions = struct { + /// Address space for raw integer addresses; typed pointers retain theirs. + space: NVPTXAddressSpace = .global, + level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, + }; + + /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. + pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { + if (@TypeOf(value) != Value) { + const n = common.arity(@TypeOf(value), "store: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; + if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + var args: [9]*const mlir.Value = undefined; + args[0] = address.inner; + for (words, 0..) |word, i| args[i + 1] = word.inner; + _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + return; + } + self.store(addr, self.joinWords(n, words), opts); + return; + } + const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { + const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); + break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } else addr; + self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ + .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), + }, self.loc())); + } + + pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { + return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; + } + + pub fn clock(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); + } + + pub fn globalTimer(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); + } + // ==================== scalars ==================== - /// `value` as a constant of `dt`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dt`'s width; anything else - /// panics. + /// `value` as a constant of `dt`; a Value passes through and enums use + /// their backing integers. An integer is signed or an unsigned bit + /// pattern of `dt`'s width; anything else panics. pub fn constant(self: *Builder, dt: DType, value: anytype) Value { if (@TypeOf(value) == Value) return value; + if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); const ctx = self.ctx; const T = @TypeOf(value); if (isFloatDtype(dt)) { @@ -2957,7 +3751,7 @@ test "guard, shared memory, sync, layouts, for loop" { const i = b.blockIdx().x.mul(128).add(tid); smem.set(.{tid}, gx.get(.{i})); b.syncThreads(); - b.namedBarrier(2, 128); + b.barrierCtaSyncAligned(2, 128); b.launchDependents(); b.waitForDependency(); const j = b.constant(.i32, 127).sub(tid); @@ -2976,7 +3770,9 @@ test "guard, shared memory, sync, layouts, for loop" { try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); - try expectContains(kernel, "bar.sync 2, 128;"); + try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(kernel, "value = 2 : i32"); + try expectContains(kernel, "value = 128 : i32"); try expectContains(kernel, "griddepcontrol.launch_dependents;"); try expectContains(kernel, "griddepcontrol.wait;"); try expectContains(kernel, "scf.if"); @@ -3113,30 +3909,1045 @@ test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); defer b.deinit(); - const descriptor = try cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(ctx, .{ - .kind = .sm_90, - .valType = DType.f8e4m3fn.toMlir(ctx), - .copyBits = 16384, - .tmaGbasis = b.layoutType(b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) })), - .tmaFormat = .U8, + const tma: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 16384, + .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), + .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), + .coordinate_rank = 3, + }; + var pair_tma = tma; + pair_tma.cta_group = .two; + const args = try b.declareArgs(.{ + .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, }); - _ = try b.declareArgs(.{ .descriptor = .{ .mlir_type = .{ .type_ = descriptor.type_(), .grid_constant = true } } }); + _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); + _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); + for ([_]CtaGroup{ .one, .two }) |group| { + _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); + } + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); const ir = try b.finish(.{ 192, 1, 1 }); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x128x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); + try expectContains(ir, "non_exec_tiled_tma_load"); + try expectContains(ir, "value = 3 : i32"); + try expectContains(ir, "value = 32 : i32"); + try expectContains(ir, "!llvm.ptr<7>"); + try expectContains(ir, "#nvvm.mem_scope"); + try expectContains(ir, "#nvvm.mem_scope"); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "mbarrier waits preserve acquire scope through polling and blocking retries" { + const ctx = try testContext(); + defer ctx.deinit(); + for ([_]MBarrierScope{ .cta, .cluster }) |scope| { + var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + const barrier = b.allocSmem(.i64, 1, 8); + const phase = b.constant(.i32, 0); + _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); + _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); + b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); + b.mbarrierWait(barrier, phase, .{ .scope = scope }); + + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try expectContains(ir, "value = 1234 : i32"); + try expectContains(ir, "\"scf.if\""); + switch (scope) { + .cta => { + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); + }, + .cluster => { + try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + }, + } + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + } +} + +test "SM100 raw primitives preserve operand types and round-trip through MLIR" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 1024, + .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const a = try b.declareArgs(.{ + .gmem = .{ .ptr = DType.i32 }, + .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + }); + const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); + const desc = b.tmaDescriptorAddress(atom); + const dst = b.constant(.i32, 1024); + const bar = b.constant(.i32, 0); + const zero = b.constant(.i32, 0); + const gmem = b.ptrToInt(a.gmem, .i64); + const pred = b.threadIdx().x.eq(0); + b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); + b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); + b.prefetchL1(gmem); + b.prefetchTensorMap(desc); + const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); + const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); + b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); + b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); + b.setMaxRegisterIncrease(128); + b.barrierCtaSyncAligned(0, 512); + b.setMaxRegisterDecrease(128); + b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); + b.mbarrierSpin(bar, zero, .cta); + b.mbarrierSpin(bar, zero, .cluster); + const state = b.mbarrierArrive(bar, .{}).?; + const remote = b.mapDsmemPtr(dst, zero); + _ = remote; + _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); + b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); + b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + const any = b.voteAnySync(pred, null); + const mask = b.voteBallotSync(pred, null); + const x = b.constant(.f32, 1.0); + const y = b.constant(.f32, 2.0); + const maximum = b.fmax(x, y, .{}); + const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); + const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); + const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); + const scalar = b.fma(maximum, fused[0], fused[1]); + b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); + + const ir = try b.finish(.{ 512, 1, 1 }); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", + "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", + "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", + "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", + "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", + "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", + "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", + "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", + "%globaltimer", + }) |op| try expectContains(ir, op); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { + try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); + try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); +} + +test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); + defer b.deinit(); + const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); + const addr = b.constant(.i32, 128); + const fields = [_]u64{ + 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes + 0x4002404000010000, // sw128, pattern at byte 128: base 1 + 0x400e404000010000, // sw128, pattern at byte 896: base 7 + 0x00007fff3fff0000, // unswizzled, largest representable offsets + }; + const descs = [_]Value{ + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), + }; + const out = b.ptrToInt(args.out, .i64); + for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + for (fields) |bits| { + const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); + defer std.testing.allocator.free(expected); + try expectContains(ir, expected); + } + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const n = b.blockIdx().x.add(1); + const singleton = b.constant(.i32, 1); + const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); + const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); + const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); + _ = compact; + _ = explicit; + _ = static_nested; + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "((4,?),?):((1,4),?)"); + try expectContains(ir, "(?,16):(1,?)"); + try expectContains(ir, "((4,8),16):((1,4),32)"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves 64-bit shape and stride values" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); + defer b.deinit(); + const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); + const wide = args.extent.add(@as(i64, 1) << 32); + const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); + const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; + try std.testing.expect(explicit.type_().eql(b.layoutType(.{ + .shape = b.expr(.{ 8, 2 }), + .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, + }))); + _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); + try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); + try expectContains(ir, "value = 4294967296 : i64"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const args = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + }); + const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); + const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); + const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); + b.copy(store.atom, .{smem}, .{tile}, .{}); + b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); + const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); + try expectContains(ir, "(16,4):(1@0,1@1)"); + try expectContains(ir, "tma_store"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); + const tid = b.threadIdx().x; + const mbar = b.allocSmem(.i64, 1, 8); + b.barrier(.{}); + b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); + b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); + _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); + const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; + b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); + b.clusterArriveRelaxed(.{}); + b.clusterWait(); + b.clusterArriveRelaxed(.{ .aligned = true }); + b.barrierClusterWaitAligned(); + _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); + _ = b.voteBallotSync(tid.eq(0), 0xff); + const x = tid.to(.f32); + const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); + const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); + b.store(a.out, state.to(.i32), .{}); + b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); + const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); + const raw_desc = b.smemDescriptorToInt(desc); + _ = b.intToSmemDescriptor(raw_desc); + const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); + const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; + try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); + try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); + b.store(mapped, tid, .{}); + const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; + for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); + const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); + b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); + b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); + try expectContains(ir, "nvvm.store.ext"); + try expectContains(ir, "vector<2xi32>"); + try expectContains(ir, "#nvvm.fp_rnd_mode"); + try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + .mode = .gather4, + }; + var pair = cfg; + pair.cta_group = .two; + const args = try b.declareArgs(.{ + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, + }); + const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); + const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); + const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; + const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); + const mbar = b.allocSmem(.i64, 1, 8); + b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ + .tma_bar_ptr = mbar, + .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), + }); + b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); + b.endFunction(.{ 32, 1, 1 }); + + b.beginFunction("gather4_copy", .host); + const host = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + }); + const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const tiler = b.makeTile(&.{ 16, 4 }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); + b.returnHostStatus(b.constant(.i32, 0)); + b.endFunction(null); + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); + try expectContains(ir, "atom.tma_partition"); + try expectContains(ir, "mode = gather4, num_cta = 1"); + try expectContains(ir, "mode = gather4, num_cta = 2"); + try expectContains(ir, "non_exec_2d_gather4_tma_load", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); + const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); + const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); + for (out, 0..) |*o, i| { + o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{packed_.inner} }, + .results = .{ .flat = &.{elem} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, + .location = b.loc(), + })); + } + return out; +} + +pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { + var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); + for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); + return w.written(); +} + +pub fn ldAcq(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); +} + +pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); +} + +pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { + return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); +} + +pub fn fenceAcqRelCta(b: *Builder) void { + asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); +} + +pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return b.constant(dt, value); +} + +/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand +/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. +pub fn zext64(b: *Builder, v: Value) Value { + if (v.dtype() == .i64) return v; + return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); +} + +pub fn i16Of(b: *Builder, v: Value) Value { + return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); +} + +pub fn cst32(b: *Builder, v: anytype) Value { + return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); +} + +pub fn cst64(b: *Builder, v: u64) Value { + return b.constant(.i64, @as(i64, @bitCast(v))); +} + +pub fn getSmIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); +} + +pub fn getLaneIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); +} + +/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. +pub fn getGridIdx(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); +} + +pub fn blockRankInCluster(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); +} + +/// `clock64()` (`mov.u64 %rd, %clock64;`). +pub fn clock64(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); +} + +/// Keep the original election sequence until typed election matches its small-batch performance. +pub fn electOneSync(b: *Builder) Value { + const pred = asmValue(b, + \\{ + \\.reg .b32 %rx; + \\.reg .pred %px; + \\ mov.u32 $0, 0; + \\ elect.sync %rx|%px, $1; + \\@%px mov.s32 $0, 1; + \\} + , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); + return pred.ne(0); +} + +/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). +pub fn trap(b: *Builder) void { + asmVoid(b, "trap;", &.{}, "~{memory}"); +} + +/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may +/// reach it at different program points). +pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { + asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); +} + +/// `cute::cluster_arrive`. +pub fn clusterArrive(b: *Builder) void { + asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); +} + +/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). +pub fn gridDependencySynchronize(b: *Builder) void { + asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::warpgroup_reg_alloc`. +pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `cutlass::arch::warpgroup_reg_dealloc`. +pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `ptx::ld_volatile(const uint64_t*)`. +pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); +} + +/// `ptx::ld_volatile(const uint32_t*)`. +pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const uint32_t*)`. +pub fn ldSharedU32(b: *Builder, smem: Value) Value { + return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const float2*)`. +pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { + return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); +} + +/// `ptx::ld_shared(const float4*)`. +pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); +} + +/// `ptx::ld_shared(const uint4*)`. +pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); +} + +/// `ptx::st_shared(const void*, x, y, z, w)`. +pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { + asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); +} + +/// `ptx::st_shared(const uint32_t*, uint32_t)`. +pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { + asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); +} + +/// `ptx::st_shared(const float2*, float2)`. +pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { + asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); +} + +/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. +pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { + std.debug.assert(num_bytes % 8 == 0); + asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); +} + +/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. +pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { + return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); +} + +/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte +/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its +/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). +pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { + std.debug.assert(words.len % 4 == 0); + const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); + const mapped_dst = mapaShared(b, dst, dst_cta_idx); + var i: usize = 0; + while (i < words.len) : (i += 4) { + asmVoid( + b, + "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", + &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, + "r,r,r,r,r,r,~{memory}", + ); + } +} + +/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. +pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); +} + +/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. +pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); +} + +/// `ClusterBarrier::arrive()` on this CTA's barrier. +pub fn mbarrierArrive(b: *Builder, bar: Value) void { + asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); +} + +/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of +/// cluster CTA `cta_id`. +pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { + asmVoid( + b, + "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", + &.{ bar, cta_id }, + "r,r,~{memory}", + ); +} + +/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA +/// `cta_id`'s barrier, only when `pred == 1`. +pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { + asmVoid( + b, + "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", + &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, + "r,r,r,r,~{memory}", + ); +} + +/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). +pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { + asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); +} + +/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of +/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. +pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, + "r,l,r,r,r,l,~{memory}", + ); +} + +/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. +pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, + "r,l,r,r,r,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of +/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. +pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, + "r,l,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. +pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, + "l,r,r,l,~{memory}", + ); +} + +/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. +pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { + asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. +pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { + asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. +pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { + asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `ptx::tcgen05_before_thread_sync`. +pub fn tcgen05BeforeThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); +} + +/// `ptx::tcgen05_after_thread_sync`. +pub fn tcgen05AfterThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). +pub fn fenceViewAsyncTmemLoad(b: *Builder) void { + asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); +} + +/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so +/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). +pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { + comptime var constraints: []const u8 = ""; + inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; + inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); + constraints = constraints ++ ",~{memory}"; + return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); +} + +/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks +/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value +/// tied through it. +pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { + const n = values.len; + const a = b.arena.allocator(); + var w: std.Io.Writer.Allocating = .init(a); + for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); + for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); + w.writer.writeAll(",~{memory}") catch @panic("OOM"); + return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); +} + +/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets +/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. +pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { + return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); +} + +/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. +pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { + asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); +} + +/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, +/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) +/// nonzero accumulates. +pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { + const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); + asmVoid( + b, + "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", + &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, + "r,l,l,r,r,r,r", + ); +} + +/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, +/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. +pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { + var elected = b.openIf(electOneSync(b)); + { + asmVoid( + b, + "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", + &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, + "r,h,~{memory}", + ); + elected.close(); + } +} + +pub fn bits32(v: Value) Value { + return if (v.dtype() == .i32) v else v.bitCast(.i32); +} + +pub fn fromBits32(v: Value, dt: DType) Value { + return if (dt == .i32) v else v.bitCast(dt); +} + +/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). +pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { + const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. +pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { + const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_up_sync(0xffffffff, v, delta)`. +pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { + const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__reduce_add_sync(0xffffffff, v)` (u32). +pub fn reduceAddSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__reduce_min_sync(0xffffffff, v)` (u32). +pub fn reduceMinSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). +pub fn allSync(b: *Builder, pred: Value) Value { + const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); + return r.ne(0); +} + +pub fn popc(b: *Builder, v: Value) Value { + return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; +} + +/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). +pub fn ffs(b: *Builder, v: Value) Value { + return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); +} + +/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. +pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { + return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); +} + +/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). +pub fn fminf(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +/// `fabsf(x)` (`abs.f32`). +pub fn fabsf(b: *Builder, x: Value) Value { + return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); +} + +/// `math::fast_rcp(x)`. +pub fn fastRcp(b: *Builder, x: Value) Value { + return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); +} + +/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). +pub fn expfFast(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). +pub fn expfFastNeg(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. +pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { + return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); +} + +/// `__hmin2(a, b)` on bf16x2 words. +pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__hmax2(a, b)` on bf16x2 words. +pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). +pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { + return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); +} + +/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with +/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). +pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { + const lo = i16Of(b, bf16x2); + const hi = i16Of(b, bf16x2.shrLogical(16)); + return .{ + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), + }; +} +/// IEEE operations used when staging quantized inputs; no FTZ or contraction. +pub fn divRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} +pub fn mulRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +test "canonical primitives preserve aligned barriers and runtime transaction counts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const bar = b.constant(.i32, 0); + const phase = b.constant(.i32, 0); + const bytes = b.threadIdx().x.add(16); + b.barrierCtaSyncAligned(1, 32); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + b.mbarrierInit(bar, 1); + b.fenceMbarrierInit(); + b.mbarrierArriveExpectTx(bar, bytes); + b.mbarrierArriveExpectTx(bar, 16); + b.mbarrierWait(bar, phase, .{ .test_first = false }); + _ = phase.bitXor(1); + b.fenceProxyAsyncShared(); + b.fenceProxyAsyncShared(); + b.cpAsyncBulkCommitGroup(); + b.cpAsyncBulkWaitGroup(0, false); + _ = b.voteBallotSync(bytes.ne(0), null); + _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); + try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/dsl/fly/README.md b/dsl/fly/README.md index b886988b..687c99dc 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel.zig`. +and `zml/kernel/fly/test.zig`. Model kernels live beside their consumers: diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index ec1ae61f..b01b7189 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,8 +104,7 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. Call it inside - `zml.ops.manualComputation` so it sees each partition's shards. + the default `.broadcast` keeps a copy per chip. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index b78e230b..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -65,14 +65,14 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - try exe.call(io, exe_args, &exe_results, .{}); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module const run_start: std.Io.Timestamp = .now(io, .awake); - try exe.call(io, exe_args, &exe_results, .{}); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index adc3882b..d471f74d 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -9,10 +9,3 @@ zig_binary( deps = ["//zml"], visibility = ["//visibility:public"], ) - -# Tooling: the loader's validation harness. Its header lists its knobs. -zig_binary( - name = "load_check", - main = "load_check.zig", - deps = ["//zml"], -) diff --git a/examples/io/load_check.zig b/examples/io/load_check.zig deleted file mode 100644 index ad98ce6a..00000000 --- a/examples/io/load_check.zig +++ /dev/null @@ -1,349 +0,0 @@ -//! Loads every tensor of a checkpoint with `zml.io.Loader` and checks the -//! bytes the devices hold against the checkpoint's. Tooling, not an -//! example: the harness the loader's backends are validated with on each -//! host. -//! -//! bazel run //examples/io:load_check -- [replicated|sharded|tp] -//! bazel run //examples/io:load_check -- dma-bench -//! -//! The checkpoint is a model directory, a .safetensors file or a -//! .safetensors.index.json, local or remote (hf://, s3://, gs://, https://, -//! with the credentials and proxies of the environment). Each tensor is -//! placed on the `tp` mesh, all the platform's devices on its `model` axis, -//! by the mode: -//! - `replicated`: whole on every device; -//! - `sharded`, the default: split along its first axis, which the devices -//! must divide, whatever the model (DeepSeek-V4.x cannot bind on 2 or 4 -//! devices); -//! - `tp`: the input axis of `o_proj` and `down_proj` weights, as tensor -//! parallelism shards them (column-sharded: every device's bytes are -//! strided in the checkpoint), the first axis of the others. -//! A scalar is replicated in every mode. -//! -//! `dma-bench` loads nothing: it runs the direct backend's DMA calibration -//! (`zml.io.dma_calibration.measure`) on the platform and logs the -//! block it selects. A platform that loads through the buffered backend has -//! no DMA to measure, and CPU selects the smallest block unmeasured. Its -//! knobs, beside the platform's below, are the screen's options, at their -//! defaults when unset: -//! - `ZML_DMA_BENCH_BLOCK_MIB=a,b,...`: the candidate blocks in MiB, each -//! at most 32 (2, 4, 8, 16 and 32); a load takes a single one (below); -//! - `ZML_DMA_BENCH_BLOCK_PARALLELISM=n`: transfers in flight per device, -//! at most 32 (8); -//! - `ZML_DMA_BENCH_WINDOW_MS=n`: a screen window's minimum duration (2); -//! - `ZML_DMA_BENCH_MIN_TRANSFERS=n`: the transfers a window waits for on -//! the measured device, 0 for none (32); -//! - `ZML_DMA_BENCH_BLOCK_TOLERANCE=f`: the smallest block within this -//! fraction of the peak rate is selected (0.08). -//! -//! The loader logs the bytes it loaded, its time and rate at info, and its -//! stats at debug, on the `zml/io/load` scope; `zml/io` logs the DMA -//! calibration's pick with its measured rate on one device (`dma_bench`); -//! `zml/vfs` logs the remote read totals. This tool enables all three. -//! -//! Knobs, from the environment: -//! - `ZML_LOAD_CHECK=n`: once loaded, read every n-th tensor and the largest -//! one back to host and compare them with the checkpoint's bytes, every -//! replica of a tensor each device holds whole. 1 checks every tensor; -//! unset or 0, none. A difference is `LoadContentMismatch`, logged. -//! - `ZML_LOAD_READ_PARALLELISM=n`: the loader's concurrent source reads -//! (`Loader.Options.read_parallelism`); unset, the checkpoint's default -//! width, 16 locally and 32 remotely. -//! - `ZML_DMA_BENCH_BLOCK_MIB=n`: the direct backend's DMA block, in MiB -//! (`Loader.Options.dma_block_size`); unset, calibration picks one. -//! - `ZML_DIRECT_IO=off|on|auto`: direct I/O for local checkpoint files -//! (`Loader.Options.direct_io`, `VFS.DirectIo`); `auto` when unset. -//! - `ZML_LOAD_MEMORY=default|host_pinned|host_mapped`: where every tensor -//! is delivered (`TensorStore.Memory`, `View.setMemory`); `default`, device -//! memory, when unset. The loader refuses a kind the platform cannot take -//! (`InvalidBinding`). -//! - `ZML_GPU_MEMORY_FRACTION=f`: a preallocated BFC pool of that fraction -//! of each device's memory on CUDA, ROCm and oneAPI, instead of the -//! platform's allocator defaults, for a run that observes the loader's -//! admission on a device with room. -//! - `ZML_CPU_DEVICE_COUNT=n`: the CPU platform's devices (4 by default). -const std = @import("std"); - -const zml = @import("zml"); - -const log = std.log.scoped(.load_check); - -pub const std_options: std.Options = .{ - .log_level = .info, - .log_scope_levels = &.{ - .{ .scope = .@"zml/io/load", .level = .debug }, - .{ .scope = .@"zml/io", .level = .debug }, - .{ .scope = .@"zml/vfs", .level = .debug }, - }, -}; - -const Mode = enum { replicated, sharded, tp }; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - var args = init.minimal.args.iterate(); - _ = args.next(); // skip program name - const path = args.next() orelse { - log.err("usage: load_check [replicated|sharded|tp], or load_check dma-bench", .{}); - return error.MissingPath; - }; - if (std.mem.eql(u8, path, "dma-bench")) return dmaBench(init); - const mode = std.meta.stringToEnum(Mode, args.next() orelse "sharded") orelse { - log.err("the mode must be replicated, sharded or tp", .{}); - return error.InvalidArgument; - }; - const check_stride = try envNumber(usize, init.environ_map, "ZML_LOAD_CHECK") orelse 0; - const direct_io = std.meta.stringToEnum(zml.io.VFS.DirectIo, init.environ_map.get("ZML_DIRECT_IO") orelse "auto") orelse { - log.err("ZML_DIRECT_IO must be off, on or auto", .{}); - return error.InvalidArgument; - }; - const dma_block_mib = try envNumber(u32, init.environ_map, "ZML_DMA_BENCH_BLOCK_MIB"); - const memory = std.meta.stringToEnum(zml.io.TensorStore.Memory, init.environ_map.get("ZML_LOAD_MEMORY") orelse "default") orelse { - log.err("ZML_LOAD_MEMORY must be default, host_pinned or host_mapped", .{}); - return error.InvalidArgument; - }; - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - var remotes: zml.io.VFS.Remotes = undefined; - try remotes.init(allocator, &vfs, init.environ_map); - defer remotes.deinit(); - - const io = vfs.io(); - - const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(init.environ_map)); - defer platform.deinit(allocator, io); - - // Returns at once: the loader sets its backend up while the - // checkpoint's headers are read and its tensors bound. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ - .checkpoint = path, - .read_parallelism = try envNumber(usize, init.environ_map, "ZML_LOAD_READ_PARALLELISM"), - .dma_block_size = if (dma_block_mib) |mib| @as(usize, mib) * zml.MiB else null, - .direct_io = direct_io, - }); - defer loader.deinit(); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - - const tensor_parallel: *const zml.Mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); - const model_axis: u3 = @intCast(tensor_parallel.resolveLogicalAxis("model").?); - const rows: zml.Sharding.Partitioning = .init(&.{.sharded(model_axis)}); - const columns: zml.Sharding.Partitioning = .init(&.{ .replicated, .sharded(model_axis) }); - - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); - defer store.deinit(); - - const Model = struct { tensors: []zml.Tensor }; - // The checkpoint tensor each one is bound to, at the same index. - const sources = store.registry.tensors.values(); - const model: Model = .{ .tensors = try allocator.alloc(zml.Tensor, sources.len) }; - defer allocator.free(model.tensors); - const view = store.view(); - for (model.tensors, store.registry.tensors.keys(), sources) |*tensor, key, source| { - const rank = source.shape.rank(); - const column_sharded = rank == 2 and - (std.mem.endsWith(u8, key, "o_proj.weight") or std.mem.endsWith(u8, key, "down_proj.weight")); - tensor.* = if (rank == 0 or mode == .replicated) - view.createReplicatedTensor(key, null) - else if (mode == .tp and column_sharded) - view.createTensor(key, null, .tp, columns) - else - view.createTensor(key, null, .tp, rows); - if (memory != .default) view.setMemory(tensor.*, memory); - } - - var progress = std.Progress.start(io, .{ .root_name = "load_check", .disable_printing = true }); - defer progress.end(); - - var buffers = try zml.mem.bufferize(allocator, &model); - defer zml.mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &store, progress); - try loader.wait(); - - if (check_stride != 0) try checkLoaded(allocator, io, sources, buffers.tensors, check_stride); -} - -/// Runs `dma-bench`: the DMA calibration screen on the platform, with the -/// `ZML_DMA_BENCH_*` knobs. -fn dmaBench(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - const environ_map = init.environ_map; - - const defaults: zml.io.dma_calibration.Options = .{}; - const window_ms = try envNumber(i64, environ_map, "ZML_DMA_BENCH_WINDOW_MS"); - const opts: zml.io.dma_calibration.Options = .{ - .block_sizes = try envMibList(init.arena.allocator(), environ_map, "ZML_DMA_BENCH_BLOCK_MIB") orelse defaults.block_sizes, - .block_parallelism = try envNumber(usize, environ_map, "ZML_DMA_BENCH_BLOCK_PARALLELISM") orelse defaults.block_parallelism, - .minimum_duration = if (window_ms) |ms| .fromMilliseconds(ms) else defaults.minimum_duration, - .minimum_transfers = try envNumber(u64, environ_map, "ZML_DMA_BENCH_MIN_TRANSFERS") orelse defaults.minimum_transfers, - .block_selection_tolerance = try envNumber(f64, environ_map, "ZML_DMA_BENCH_BLOCK_TOLERANCE") orelse defaults.block_selection_tolerance, - }; - - const platform: *zml.Platform = try .auto(allocator, io, try platformOptions(environ_map)); - defer platform.deinit(allocator, io); - - const block_size = try zml.io.dma_calibration.measure(allocator, io, platform, opts); - log.info("dma_benchmark block_bytes={d} parallelism={d}", .{ block_size, opts.block_parallelism }); -} - -/// The platform settings of the environment, which are not loader knobs. -fn platformOptions(environ_map: *const std.process.Environ.Map) !zml.platform.CreateOptions { - var options: zml.platform.CreateOptions = .{}; - if (try envNumber(u32, environ_map, "ZML_CPU_DEVICE_COUNT")) |device_count| { - options.cpu.device_count = device_count; - } - // Unset keeps the platform's allocator defaults (VMM on CUDA and ROCm); - // set, every XLA GPU target gets a preallocated BFC pool of that - // fraction, whose `bytes_limit` admission measures against. - if (try envNumber(f32, environ_map, "ZML_GPU_MEMORY_FRACTION")) |memory_fraction| { - const pool: zml.platform.CreateOptions.XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }; - options.cuda = pool; - options.rocm = pool; - options.oneapi = pool; - } - return options; -} - -/// Reads every `stride`-th of `buffers` and the largest eligible one back to -/// host and compares the bytes with the checkpoint tensor each was bound -/// to, `sources` at the same index, so a buffer that reported ready before -/// every piece landed is caught. A buffer every device holds whole is -/// compared on every replica; a partitioned one is assembled with -/// `toSliceAlloc`, which keeps the last replica of a region that several -/// devices hold. Each source file is opened once (an `hf://` open is a HEAD -/// plus a redirect). Skipped: sub-byte tensors partitioned over several -/// devices (`toSliceAlloc` places their shards by element stride). -fn checkLoaded( - allocator: std.mem.Allocator, - io: std.Io, - sources: []const zml.safetensors.Tensor, - buffers: []const zml.Buffer, - stride: usize, -) !void { - std.debug.assert(sources.len == buffers.len); - const Eligibility = struct { - fn checkable(source: zml.safetensors.Tensor, buffer: zml.Buffer) bool { - const partitioned = buffer.byteSize() / buffer.numShards() != source.byteSize(); - return !partitioned or source.shape.dtype().bitSizeOf() >= 8; - } - }; - var largest: ?usize = null; - for (sources, buffers, 0..) |source, buffer, i| { - if (!Eligibility.checkable(source, buffer)) continue; - if (largest == null or source.byteSize() > sources[largest.?].byteSize()) largest = i; - } - var files: std.StringHashMapUnmanaged(std.Io.File) = .empty; - defer { - var it = files.valueIterator(); - while (it.next()) |file| file.close(io); - files.deinit(allocator); - } - var expected: []u8 = &.{}; - defer allocator.free(expected); - var replica: []u8 = &.{}; - defer allocator.free(replica); - const started: std.Io.Timestamp = .now(io, .awake); - var checked: usize = 0; - var skipped: usize = 0; - var checked_bytes: usize = 0; - for (sources, buffers, 0..) |source, buffer, i| { - if (i % stride != 0 and i != largest) continue; - if (!Eligibility.checkable(source, buffer)) { - skipped += 1; - continue; - } - const file = files.get(source.file_uri) orelse blk: { - const opened = try std.Io.Dir.openFile(.cwd(), io, source.file_uri, .{ .mode = .read_only }); - errdefer opened.close(io); - try files.put(allocator, source.file_uri, opened); - break :blk opened; - }; - const source_bytes: usize = @intCast(source.byteSize()); - if (expected.len < source_bytes) { - allocator.free(expected); - expected = &.{}; - expected = try allocator.alloc(u8, source_bytes); - } - try source.readRange(io, file, expected[0..source_bytes], 0); - - if (buffer.byteSize() / buffer.numShards() == source_bytes) { - // Every shard holds the whole tensor: compare each replica. - if (replica.len < source_bytes) { - allocator.free(replica); - replica = &.{}; - replica = try allocator.alloc(u8, source_bytes); - } - var shards = buffer.shards(); - var shard_index: usize = 0; - while (shards.next()) |shard| : (shard_index += 1) { - try shard.toHost(io, replica[0..source_bytes]); - try expectSameBytes(source.name, shard_index, expected[0..source_bytes], replica[0..source_bytes]); - checked_bytes += source_bytes; - } - } else { - const slice = try buffer.toSliceAlloc(allocator, io); - defer slice.free(allocator); - try expectSameBytes(source.name, null, expected[0..source_bytes], slice.constData()); - checked_bytes += source_bytes; - } - checked += 1; - } - log.info("load check: ok tensors_checked={d} of {d} skipped={d} bytes={Bi:.2} elapsed={f}", .{ - checked, - sources.len, - skipped, - checked_bytes, - started.untilNow(io, .awake), - }); -} - -fn expectSameBytes(name: []const u8, replica: ?usize, expected: []const u8, actual: []const u8) !void { - if (actual.len == expected.len and std.mem.eql(u8, expected, actual)) return; - const first_bad = std.mem.indexOfDiff(u8, expected, actual) orelse @min(expected.len, actual.len); - log.err("load check: mismatch tensor={s} replica={?d} expected_bytes={d} actual_bytes={d} first_difference={d}", .{ - name, - replica, - expected.len, - actual.len, - first_bad, - }); - return error.LoadContentMismatch; -} - -/// The environment variable `name` as a comma-separated list of MiB, in -/// bytes, null when unset. An entry that does not parse is -/// `InvalidArgument`, logged. -fn envMibList(allocator: std.mem.Allocator, environ_map: *const std.process.Environ.Map, name: []const u8) !?[]const usize { - const text = environ_map.get(name) orelse return null; - var sizes: std.ArrayList(usize) = .empty; - errdefer sizes.deinit(allocator); - var entries = std.mem.tokenizeScalar(u8, text, ','); - while (entries.next()) |entry| { - const mib = std.fmt.parseInt(u32, entry, 10) catch { - log.err("{s} must be MiB counts separated by commas, not '{s}'", .{ name, text }); - return error.InvalidArgument; - }; - try sizes.append(allocator, @as(usize, mib) * zml.MiB); - } - return try sizes.toOwnedSlice(allocator); -} - -/// The environment variable `name` parsed as a `T`, null when unset. A value -/// that does not parse is `InvalidArgument`, logged. -fn envNumber(comptime T: type, environ_map: *const std.process.Environ.Map, name: []const u8) !?T { - const text = environ_map.get(name) orelse return null; - const value = switch (@typeInfo(T)) { - .int => std.fmt.parseInt(T, text, 10), - .float => std.fmt.parseFloat(T, text), - else => @compileError("envNumber: " ++ @typeName(T)), - }; - return value catch { - log.err("{s} must be a number, not '{s}'", .{ name, text }); - return error.InvalidArgument; - }; -} diff --git a/examples/io/main.zig b/examples/io/main.zig index e6132ff8..c9dc05ea 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -3,6 +3,8 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; +const log = std.log.scoped(.vfs); + pub const std_options: std.Options = .{ .log_level = .info, }; @@ -22,12 +24,34 @@ pub fn main(init: std.process.Init) !void { const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; const path = it.next() orelse return error.MissingPath; + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + + try http_client.initDefaultProxies(allocator, init.environ_map); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + defer vfs_https.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - var remotes: zml.io.VFS.Remotes = undefined; - try remotes.init(allocator, &vfs, init.environ_map); - defer remotes.deinit(); + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + try vfs.register("gs", gcs_vfs.io()); const io = vfs.io(); @@ -108,13 +132,9 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.tensors.iterator(); - while (registry_it.next()) |kv| { - const name = kv.key_ptr.*; - const tensor = kv.value_ptr.*; - + for (registry.tensors.values()) |tensor| { var current = root; - var parts = std.mem.tokenizeScalar(u8, name, '.'); + var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -138,11 +158,6 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - // Returns at once: the loader sets its backend up while the - // checkpoint's headers are read and its tensors bound. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = path }); - defer loader.deinit(); - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); @@ -152,36 +167,58 @@ pub fn main(init: std.process.Init) !void { ); const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); - // The store takes the registry's entries. - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{tensor_parallel} }); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); defer store.deinit(); const AllTensorsModel = struct { tensors: []zml.Tensor, }; - const tensors = try allocator.alloc(zml.Tensor, store.registry.tensors.count()); + const tensor_count = registry.tensors.count(); + + const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); - for (tensors, store.registry.tensors.keys(), store.registry.tensors.values()) |*tensor, key, entry| { - tensor.* = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(key, null), - .sharded => if (entry.shape.rank() > 0) - store.view().createTensor(key, null, .tp, tp_partitioning) + var load_count: usize = 0; + for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(name, null), + .sharded => if (tensor.shape.rank() > 0) + store.view().createTensor(name, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(key, null), + store.view().createReplicatedTensor(name, null), }; + load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); + progress.increaseEstimatedTotalItems(load_count); defer progress.end(); - var buffers = try zml.mem.bufferize(allocator, &model); - defer zml.mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &store, progress); - try loader.wait(); // logs "Loaded weights" + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); + defer { + for (buffers.tensors) |*b| b.deinit(); + allocator.free(buffers.tensors); + } + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .parallelism = 8, + .dma_chunks = 16, + .dma_chunk_size = 256 * zml.MiB, + }); + defer loader.deinit(); + + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); }, } } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index b5c8f3ec..b24c022a 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -53,12 +53,28 @@ pub fn main(init: std.process.Init) !void { // // Virtual File Systems // + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer http_client.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); - var remotes: zml.io.VFS.Remotes = undefined; - try remotes.init(allocator, &vfs, init.environ_map); - defer remotes.deinit(); + try vfs.register("file", vfs_file.io()); + try vfs.register("gs", gcs_vfs.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); const io = vfs.io(); @@ -91,7 +107,7 @@ pub fn main(init: std.process.Init) !void { // Defines how the model's tensors are sharded across the available devices. const meshes: models.Meshes = try .init(platform); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -100,6 +116,9 @@ pub fn main(init: std.process.Init) !void { }, }; + var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + defer model.deinit(allocator); + // // Load the model and compile it // @@ -109,29 +128,14 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); - var model: models.LoadedModel = undefined; var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - var model_buffers = buffers: { - // Returns at once: the loader sets its backend up on a task of its - // own while the model is initialized and compiled. It is freed at - // the end of this block, with its pinned host memory, before the - // chat starts. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = args.model }); - defer loader.deinit(); - - model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); - errdefer model.deinit(allocator); - - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); - errdefer compiled_model.deinit(); - - // Load buffers after the model compilation to be sure to give enough room to the autotune. - break :buffers try models.LoadedModel.loadBuffers(&model, allocator, &loader, progress, &store); - }; - defer model.deinit(allocator); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); defer compiled_model.deinit(); - defer zml.mem.deinitBufferized(allocator, &model_buffers); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); + defer model.unloadBuffers(&model_buffers, allocator); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 33e22bf3..20aa60c7 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -55,23 +55,26 @@ pub const LoadedModel = union(ModelType) { } } - /// Loads the model's weights from the store with `loader`, which logs - /// them. Free them with `zml.mem.deinitBufferized`. pub fn loadBuffers( - self: *const LoadedModel, + self: *LoadedModel, allocator: std.mem.Allocator, - loader: *zml.io.Loader, - progress: std.Progress.Node, - store: *const zml.io.TensorStore, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, ) !Buffers { + return switch (self.*) { + inline else => |*m, t| @unionInit( + Buffers, + @tagName(t), + try m.loadBuffers(allocator, io, platform, store, progress), + ), + }; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { switch (self.*) { - inline else => |*m, t| { - var buffers = try zml.mem.bufferize(allocator, &m.inner); - errdefer zml.mem.deinitBufferized(allocator, &buffers); - try loader.start(&m.inner, &buffers, store, progress); - try loader.wait(); - return @unionInit(Buffers, @tagName(t), buffers); - }, + inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), } } diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index bee197fa..9f7cb909 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -6,6 +6,8 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); +const log = std.log.scoped(.lfm); + pub const Config = struct { architectures: []const []const u8, block_auto_adjust_ff_dim: bool, @@ -70,6 +72,43 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + Model.unloadBuffers(buffers, allocator); + } + pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -125,6 +164,47 @@ pub const Model = struct { }; } + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 8, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + LmHead.unloadBuffers(&self.lm_head); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + pub fn forward( self: Model, tokens: zml.Tensor, @@ -244,6 +324,10 @@ pub const LmHead = struct { const tokens = new_tokens.convert(.u32); return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; } + + pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { + RmsNorm.unloadBuffers(&self.embedding_norm); + } }; pub const OperatorKind = enum { conv, full_attention }; @@ -342,6 +426,16 @@ pub const DecoderLayer = struct { .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), }; } + + pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + switch (self.operator) { + .conv => |*operator| ShortConv.unloadBuffers(operator), + .self_attn => |*operator| Attention.unloadBuffers(operator), + } + RmsNorm.unloadBuffers(&self.operator_norm); + RmsNorm.unloadBuffers(&self.ffn_norm); + Mlp.unloadBuffers(&self.feed_forward); + } }; pub const ConvParameters = struct { is_prefill: bool }; @@ -407,6 +501,12 @@ pub const ShortConv = struct { const output = self.out_proj.forward(y); return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; } + + pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { + Linear.unloadBuffers(&self.in_proj); + Linear.unloadBuffers(&self.out_proj); + self.kernel.deinit(); + } }; pub const Attention = struct { @@ -471,6 +571,8 @@ pub const Attention = struct { v = new_kv_cache.values(cache_index); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const mesh = zml.Compiler.current().mesh(.model); + const partitioning: zml.attention.Partitioning = .head(.model); const attn = switch (attention_parameters) { .stablehlo => zml.attention.attention( q, @@ -479,6 +581,8 @@ pub const Attention = struct { tokens_position_offset, attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }), else => zml.attention.attention( q.squeeze(.batch), @@ -487,6 +591,8 @@ pub const Attention = struct { tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }) .rename(.{ .q = .seq }) .insertAxes(.seq, .{.batch}), @@ -494,6 +600,15 @@ pub const Attention = struct { return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; } + + pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { + Linear.unloadBuffers(&self.q_proj); + Linear.unloadBuffers(&self.k_proj); + Linear.unloadBuffers(&self.v_proj); + Linear.unloadBuffers(&self.out_proj); + RmsNorm.unloadBuffers(&self.q_layernorm); + RmsNorm.unloadBuffers(&self.k_layernorm); + } }; pub const Cache = struct { @@ -504,6 +619,11 @@ pub const Cache = struct { return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } + pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { + ConvCache.unloadBuffers(&self.conv); + KvCache.unloadBuffers(&self.kv); + } + pub fn reuseBuffer(self: Cache, other: Cache) Cache { return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; } @@ -524,6 +644,10 @@ pub const ConvCache = struct { return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } + pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { + self.state.deinit(); + } + pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { return .{ .state = self.state.reuseBuffer(other.state) }; } @@ -542,6 +666,11 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } + pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + } + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; } @@ -577,6 +706,11 @@ pub const Linear = struct { return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; } + pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { + linear.weight.deinit(); + if (linear.bias) |*bias| bias.deinit(); + } + pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; @@ -596,6 +730,12 @@ const Mlp = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + Linear.unloadBuffers(&self.w1); + Linear.unloadBuffers(&self.w2); + Linear.unloadBuffers(&self.w3); + } + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); } @@ -610,6 +750,10 @@ const RmsNorm = struct { return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index df617bec..ecc5abf8 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -31,7 +31,7 @@ pub const Session = struct { ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); - errdefer zml.mem.deinitBufferized(allocator, &cache_buffers); + errdefer model.Cache.unloadBuffers(&cache_buffers); var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); @@ -62,7 +62,7 @@ pub const Session = struct { pub fn deinit(self: *Session) void { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); - zml.mem.deinitBufferized(self.allocator, &self.cache_buffers); + model.Cache.unloadBuffers(&self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 4934d3e6..19b28df6 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -41,7 +41,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -49,12 +49,8 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); - defer loader.deinit(); - var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); - defer zml.mem.deinitBufferized(allocator, &model_buffers); - try loader.start(&repo_model.inner, &model_buffers, &store, progress); - try loader.wait(); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { .stablehlo => .{ .stablehlo = {} }, @@ -89,7 +85,7 @@ pub fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer activation_store.deinit(); var ctx = TestContext{ @@ -179,7 +175,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - try exe.call(self.io, args, &res, .{}); + exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -308,12 +304,16 @@ const TestContext = struct { }; fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { - const tensor = store.view().get(key) orelse return error.NotFound; + const shape = store.view().getShape(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, tensor.byteSize()); + const host_bytes = try allocator.alloc(u8, shape.byteSize()); defer allocator.free(host_bytes); - try tensor.read(io, host_bytes); + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.view().getReader(key, io, &io_buffer); + defer reader.deinit(); - return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 0dfcbb18..8a6a09e9 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -6,6 +6,8 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); +const log = std.log.scoped(.llama); + pub const Config = struct { bos_token_id: u32, eos_token_id: EosTokens, @@ -67,6 +69,43 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&buffers.model, allocator); + } + pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -122,6 +161,43 @@ pub const Model = struct { self.model.deinit(allocator); } + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&self.model, allocator); + } + fn lmHead(self: Model) LmHead { return .init(self); } @@ -194,6 +270,15 @@ const Llama = struct { allocator.free(self.layers); } + pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + RmsNorm.unloadBuffers(&self.norm); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward( @@ -244,7 +329,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -311,6 +396,13 @@ pub const TransformerLayer = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + SelfAttn.unloadBuffers(&self.self_attn); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + Mlp.unloadBuffers(&self.mlp); + } + pub const Input = struct { layer: TransformerLayer, hidden: zml.Tensor, @@ -369,6 +461,10 @@ const RmsNorm = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -390,6 +486,10 @@ const Mlp = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const proj = self.up_proj.forward(x, x.dtype()); var output = self.gate_proj.forward(x, x.dtype()); @@ -431,6 +531,10 @@ const SelfAttn = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + /// Self zml.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. @@ -483,6 +587,8 @@ const SelfAttn = struct { token_index, attention_metadata, attention_parameters, + zml.Compiler.current().mesh(.model), + .head(.model), ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index dae0d054..1b5894f1 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &meshes.all() }); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); @@ -47,12 +47,8 @@ pub fn main(init: std.process.Init) !void { var progress = std.Progress.start(io, .{ .root_name = args.model }); - var loader: zml.io.Loader = try .init(allocator, io, platform, .{}); - defer loader.deinit(); - var model_buffers = try zml.mem.bufferize(allocator, &repo_model.inner); - defer zml.mem.deinitBufferized(allocator, &model_buffers); - try loader.start(&repo_model.inner, &model_buffers, &store, progress); - try loader.wait(); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); @@ -70,7 +66,7 @@ fn run( var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = meshes }); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); defer activation_store.deinit(); try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); @@ -102,7 +98,7 @@ fn testLayer( ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); - const in_shape = (activation_store.get(in_key) orelse return error.NotFound).shape; + const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); @@ -130,7 +126,7 @@ fn testLayer( var res = try exe.results(allocator); defer res.deinit(allocator); - try exe.call(io, args, &res, .{}); + exe.call(io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -138,12 +134,16 @@ fn testLayer( } fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { - const tensor = store.get(key) orelse return error.NotFound; + const shape = store.getShape(key) orelse return error.NotFound; - const host_bytes = try allocator.alloc(u8, tensor.byteSize()); + const host_bytes = try allocator.alloc(u8, shape.byteSize()); defer allocator.free(host_bytes); - try tensor.read(io, host_bytes); + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.getReader(key, io, &io_buffer); + defer reader.deinit(); - return zml.Buffer.fromBytes(io, platform, tensor.shape, host_bytes); + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index a434c489..40174f5c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -6,6 +6,8 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); +const log = std.log.scoped(.qwen3_5); + pub const Config = struct { text_config: TextConfig, }; @@ -72,6 +74,44 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + zml.nn.Linear.unloadBuffers(&buffers.lm_head); + } + pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -113,7 +153,7 @@ pub const Model = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer const lm_head_prefix = b: { - if (store.get("lm_head.weight") != null) break :b "lm_head"; + if (store.hasKey("lm_head.weight")) break :b "lm_head"; break :b "model.language_model.embed_tokens"; }; return .{ @@ -132,6 +172,43 @@ pub const Model = struct { self.text_model.deinit(allocator); } + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -213,7 +290,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -245,6 +322,15 @@ pub const TextModel = struct { allocator.free(self.layers); } + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + } + pub fn forward( self: TextModel, tokens: zml.Tensor, @@ -315,6 +401,16 @@ pub const TransformerLayer = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Mlp.unloadBuffers(&self.mlp); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + pub fn forward( self: TransformerLayer, x0: zml.Tensor, @@ -426,6 +522,10 @@ pub const Mlp = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -477,6 +577,10 @@ pub const SelfAttn = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); @@ -553,6 +657,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -560,6 +668,8 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); @@ -694,6 +804,10 @@ pub const GatedDeltaNet = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -869,6 +983,10 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -886,6 +1004,10 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 9fa5300f..2a3b0ba8 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -100,6 +100,43 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + } + pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -151,6 +188,42 @@ pub const Model = struct { self.text_model.deinit(allocator); } + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + } pub fn forward( self: Model, tokens_: zml.Tensor, @@ -232,7 +305,7 @@ pub const TextModel = struct { config: Config, gen_options: Model.GenOptions, ) !TextModel { - const lm_head_prefix = if (store.root().get("lm_head.weight") != null) "lm_head" else "model.language_model.embed_tokens"; + const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); errdefer allocator.free(layers); @@ -258,6 +331,16 @@ pub const TextModel = struct { allocator.free(self.layers); } + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer, allocator); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } + pub fn sampler(self: TextModel) Sampler { return .{ .norm = self.norm, @@ -354,6 +437,16 @@ pub const TransformerLayer = struct { self.moe.deinit(allocator); } + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Moe.unloadBuffers(&self.moe, allocator); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; @@ -489,6 +582,10 @@ pub const SelfAttn = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const q_proj = self.q_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); @@ -544,6 +641,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -551,7 +652,9 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), - ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); @@ -591,6 +694,10 @@ pub const Mlp = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const up_projed = self.up_proj.forward(x, x.dtype()); const gate = self.gate_proj.forward(x, x.dtype()); @@ -608,14 +715,18 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .replicated, .{}), - store.maybeCreateTensor("bias", .{.expert}, .replicated, .{}), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), .d, ), .num_experts_per_tok = num_experts_per_tok, }; } + pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { + zml.Buffer.deinitAll(Router, self); + } + pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const router_logits = self.router.forward(x, .f32); const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); @@ -651,8 +762,8 @@ pub const Moe = struct { return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{}), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{}), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -676,6 +787,8 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, moe_backend, + zml.Compiler.current().mesh(.experts), + .expert(.experts), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, @@ -692,6 +805,11 @@ pub const Moe = struct { return moe_output.add(shared); } + + pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { + _ = allocator; + zml.Buffer.deinitAll(Moe, self); + } }; pub const TextRotaryEmbedding = struct { @@ -817,6 +935,10 @@ pub const GatedDeltaNet = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + fn recurrentGatedDeltaRule( query: zml.Tensor, key: zml.Tensor, @@ -982,6 +1104,10 @@ pub const RmsNorm = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const weight_f32 = self.weight.convert(.f32); @@ -999,6 +1125,10 @@ pub const RmsNormGated = struct { return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 878b269b..f5842af8 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,6 +37,32 @@ const Mnist = struct { }; } + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer unloadBuffers(&buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); @@ -54,32 +80,24 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; + const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - const io = vfs.io(); + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - // Returns at once: the loader calibrates while the model is compiled. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); - defer loader.deinit(); - - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); - defer store.deinit(); - // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); const mnist_model: Mnist = .init(store.view()); // // Compile model @@ -92,11 +110,16 @@ pub fn main(init: std.process.Init) !void { }; defer exe.deinit(); - // Load weights - var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); - defer zml.mem.deinitBufferized(allocator, &mnist_buffers); - try loader.start(&mnist_model, &mnist_buffers, &store, .none); - try loader.wait(); // logs "Loaded weights" + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer Mnist.unloadBuffers(&mnist_buffers); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -124,7 +147,7 @@ pub fn main(init: std.process.Init) !void { printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); - try exe.call(io, args, &results, .{}); + exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index 76684d94..b7805e49 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -2,9 +2,11 @@ //! Separated to kept `mnist.zig` minimal const std = @import("std"); + const zml = @import("zml"); const base = @import("mnist.zig"); + const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ @@ -31,17 +33,42 @@ const Mnist = struct { }; } + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer zml.Buffer.deinitAll(Mnist, &buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); defer compiler.popLocation(); x = layer.forward(x); } - return x.argMax(.d).indices.convert(.u8).withPartitioning(input.shape()._sharding.mesh.?, .{ .b = .data }).toMemory(.host_pinned); + + return x.argMax(.d).indices.convert(.u8) + .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) + .toMemory(.host_pinned); } const Layer = struct { @@ -49,7 +76,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - log.warn("layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); const x = input.dot(self.weight, .d); return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); } @@ -59,16 +86,16 @@ const Mnist = struct { pub fn main(init: std.process.Init) !void { const arena = init.arena; const allocator = init.gpa; + const io = init.io; // Parse program args const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Local paths. `zml.io.VFS.Remotes` adds hf://, s3://, gs:// and https://. - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - const io = vfs.io(); + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); @@ -81,25 +108,20 @@ pub fn main(init: std.process.Init) !void { &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }, )); - log.info("topology used: {f}", .{dp_mp}); - - // Returns at once: the loader calibrates while the model is compiled. - var loader: zml.io.Loader = try .init(allocator, io, platform, .{ .checkpoint = model_path }); - defer loader.deinit(); + log.info("{f}", .{dp_mp}); var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{ .meshes = &.{dp_mp} }); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); - // // Compile model + // Compile model const bs: u32 = 4; const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + var exe = exe: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); @@ -113,17 +135,22 @@ pub fn main(init: std.process.Init) !void { .{ .program_name = "mnist", .meshes = &.{dp_mp}, - .xla_dump_to = "/tmp/zml/mnist", + .xla_dump_to = "/tmp/zml/mnist_dp", }, ); }; defer exe.deinit(); - // Load weights - var mnist_buffers = try zml.mem.bufferize(allocator, &mnist_model); - defer zml.mem.deinitBufferized(allocator, &mnist_buffers); - try loader.start(&mnist_model, &mnist_buffers, &store, .none); - try loader.wait(); // logs "Loaded weights" + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -159,7 +186,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ mnist_buffers, input_buffer }); try profiler.start(); - try exe.call(io, args, &results, .{}); + exe.call(io, args, &results, .{}); if (try profiler.stop()) |report| { log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); } diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 29de9597..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -30,14 +30,18 @@ const DemoModel = struct { } pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { - const x = input.convert(.f32); + const compiler = zml.Compiler.current(); + const x = input.convert(.f32); var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); + const gate = y.scale(0.01).sigmoid(); + + const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); + return zml.ops.manualComputation( (struct { y: zml.Tensor, @@ -50,6 +54,11 @@ const DemoModel = struct { }).body, .{ .y = y, .gate = gate }, y.shape(), + compiler.mesh(.dp_mp), + .{ + .in_specs = .{ .y = spec, .gate = spec }, + .out_specs = spec, + }, ); } }; @@ -60,7 +69,7 @@ fn usage() void { \\ \\Simple sharding demo: \\ - explicit partitioning on data/model axes - \\ - shard-local tensor print + \\ - tensor print \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: @@ -242,7 +251,7 @@ pub fn main(init: std.process.Init) !void { defer exe_results.deinit(allocator); exe_args.set(.{ w_buf, b_buf, input_buf }); - try exe.call(io, exe_args, &exe_results, .{}); + exe.call(io, exe_args, &exe_results, .{}); var out = exe_results.get(zml.Buffer); defer out.deinit(); diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc index fe1296db..6e7c296d 100644 --- a/mlir/dialects/cute_ir/capi.cc +++ b/mlir/dialects/cute_ir/capi.cc @@ -166,6 +166,10 @@ MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { return getAlgebraValue(attr); } +intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { + return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); +} + MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { return getStaticAttr(context, value); } diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h index e9e285d2..002f4885 100644 --- a/mlir/dialects/cute_ir/capi.h +++ b/mlir/dialects/cute_ir/capi.h @@ -54,6 +54,7 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( MlirContext context, intptr_t numElements, MlirAttribute const *elements); MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 8a031d0b..b277d061 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -145,6 +145,10 @@ pub const ShapeAttr = opaque { return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); } + pub fn rank(self: *const ShapeAttr) usize { + return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); + } + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { return @ptrCast(self); } diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index b66b7abe..9673870f 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index 78524e0a..d1197317 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -22,6 +22,7 @@ pub const elementtype = ops.elementtype; pub const AtomicBinOp = ops.AtomicBinOp; pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; +pub const ptrtoint = ops.ptrtoint; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; pub const extractvalue = ops.extractvalue; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 2102f68a..d1fb5771 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -234,6 +234,15 @@ pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *co }); } +/// llvm.ptrtoint — the integer address of an LLVM pointer. +pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "llvm.bitcast", .{ diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index 28541d44..6ba84b4d 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -20,6 +20,7 @@ const operation_names: []const []const u8 = &.{ "llvm.intr.lifetime.end", "llvm.intr.lifetime.start", "llvm.inttoptr", + "llvm.ptrtoint", "llvm.load", "llvm.mlir.poison", "llvm.mlir.undef", diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig index d46f3a4f..14a0e8e2 100644 --- a/mlir/dialects/nvvm/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -136,7 +136,8 @@ pub fn mbarrier_wait_parity( } /// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) -/// of the mbarrier completes, suspending up to `suspend_time` (i32) cycles per try. +/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) +/// is a suspend-time hint in nanoseconds per try. pub fn mbarrier_try_wait_parity( ctx: *mlir.Context, ptr: *const mlir.Value, @@ -147,6 +148,27 @@ pub fn mbarrier_try_wait_parity( return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); } +/// One potentially suspending parity test with acquire semantics. A false +/// result means the timeout expired; callers that need completion must retry. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity_timelimit( + ctx, + ptr, + parity, + suspend_time, + .int(ctx, .i1), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, + location, + ); +} + /// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return ops.fence_mbarrier_init(ctx, location); @@ -208,10 +230,12 @@ pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued -/// tcgen05 operations complete. -pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, .{ .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the +/// tcgen05 MMA/copy operations issued by the executing thread with `group` +/// complete. An optional mask selects CTAs in the cluster whose barriers at +/// the same shared-memory offset receive one arrival each. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); } /// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 499e753c..3715361f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -484,6 +484,11 @@ pub const Client = opaque { return @ptrCast(ret.executable.?); } + pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { + var future = io.async(compileRaw, .{ self, api, args }); + return future.await(io); + } + pub const BufferFromHostBufferArgs = struct { data: [*]const u8, buffer_type: BufferType, @@ -672,19 +677,8 @@ pub const Client = opaque { }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { - return self.createUninitializedBufferThrough(api, args, Api.call); - } - - /// As `createUninitializedBuffer`, for a request the caller expects to - /// fail: its error is logged at debug level only. - pub fn createUninitializedBufferSilent(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { - return self.createUninitializedBufferThrough(api, args, Api.callSilent); - } - - /// `call` is `Api.call` or `Api.callSilent`. - fn createUninitializedBufferThrough(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs, comptime call: anytype) ApiError!*Buffer { var layout = args.layout.toCStruct(); - const ret = try call(api, .PJRT_Client_CreateUninitializedBuffer, .{ + const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 1105f776..08817afe 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index ced8ae98..8482e52a 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "77d2ddb19566604c827828fe3dcc53e938f3a1b16d431416fdd5c466266f5242", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-amd64.tar.zst", + "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "2cd18393c44816d3d01aa22fbd890632ba46eca3939c644bddef00d590578a80", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-arm64.tar.zst", + "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 0aa3905d..afb26864 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610221403.283.1.d9f4f68759e1/zml-furiosa-linux-amd64.tar.zst", - sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ebbd9e3c..ee33fd58 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,7 +50,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 03cdaf02..71e1858a 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "6eee91776397d472ac590413cf44d4ff3f7b31bebb163aa96339c6becaf2defd", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-metal-darwin-arm64.tar.zst", + sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..a3c968d3 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index a3e69ad4..140ce102 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 272bc02c..5a829ac5 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,9 +15,11 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above. + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. - channel: noble main - url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z archs: - "amd64" @@ -43,9 +45,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - "libze1" - "zlib1g" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index babd73ab..41b01b60 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cd6c3c8673ce690a0fbf89dab5a16e6893f0f501bb2b41997c657cf6effa7549" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" def _rocm_package_name(name): return name + _ROCM_VERSION @@ -73,6 +73,7 @@ def _rocm_runfile_build_files(loaded_packages): for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", "lib/libMIOpenCKGroupedConv_*.so", "share/miopen/db/**", ]) diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 6fcda16b..ee31bd83 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -74,142 +74,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); - const api = try pjrt.Api.loadFrom(lib_path); - loaded_hip = Hip.loaded(); - break :blk api; + break :blk .loadFrom(lib_path); }; } - -/// Whether `releaseRuntimeDeviceMemory` asks the devices for their whole -/// memory. The first call that finds the HIP runtime missing, or a device -/// that does not refuse, logs it and clears this for good. It lives here, -/// not with a PJRT client: the HIP runtime the plugin loaded, and the -/// memory it keeps, belong to the process and serve every client. -var releases_runtime_memory: std.atomic.Value(bool) = .init(true); -/// The HIP calls of `releaseRuntimeDeviceMemory`, resolved by `load`, which -/// zml calls once per process; null when `libamdhip64.so.7` is not loaded -/// or lacks one of them. -var loaded_hip: ?Hip = null; - -/// Makes the HIP runtime the plugin loaded give back the device memory it -/// keeps outside the plugin's allocator, chiefly the scratch of kernels -/// that spill registers, which ROCr keeps bound to its queues and in its -/// caches until the driver refuses one of its allocations. It asks every -/// device in turn for its whole memory, rounded down to 2 MiB: the driver -/// refuses, ROCr reclaims the scratch of the device's queues and frees its -/// caches, and HIP answers `hipErrorOutOfMemory`. Kernels that spill later -/// map their scratch again. Every queue in the process that holds scratch -/// on the device is suspended and resumed, and the call waits for its -/// kernels using scratch while holding the device's ROCr memory lock: -/// running work and allocations on that device stall meanwhile, a stall, -/// not a deadlock. Restores the thread's HIP device, and clears any HIP -/// error left on the thread, earlier ones included. -/// -/// Returns whether every device refused. A missing `libamdhip64.so.7` or -/// symbol, or a device that does not refuse (a grant is freed at once), is -/// logged once and ends the trim for good: the remaining devices are not -/// asked, nor any device in later calls. -pub fn releaseRuntimeDeviceMemory(hip_ordinals: []const c_int) bool { - if (comptime !isEnabled() or builtin.os.tag != .linux) return false; - if (!releases_runtime_memory.load(.monotonic)) return false; - const hip = loaded_hip orelse { - if (releases_runtime_memory.swap(false, .monotonic)) { - log.warn("The HIP runtime (libamdhip64.so.7) is not loaded or lacks a symbol; the device memory it keeps is no longer given back", .{}); - } - return false; - }; - - var previous: c_int = 0; - const has_previous = hip.hipGetDevice(&previous) == hip_success; - defer { - if (has_previous) _ = hip.hipSetDevice(previous); - _ = hip.hipGetLastError(); - } - - for (hip_ordinals) |ordinal| { - const answer = hip.requestWholeDevice(ordinal); - if (answer == .refused) continue; - if (releases_runtime_memory.swap(false, .monotonic)) { - log.warn("HIP device {d} did not refuse a request for its whole memory: {f}; the device memory the HIP runtime keeps is no longer given back", .{ ordinal, answer }); - } - return false; - } - return true; -} - -/// `hipError_t` values (hip_runtime_api.h). -const hip_success: c_int = 0; -const hip_error_out_of_memory: c_int = 2; - -/// The HIP calls `releaseRuntimeDeviceMemory` makes, from the HIP runtime -/// already in the process. -const Hip = struct { - hipGetDevice: *const fn (device: *c_int) callconv(.c) c_int, - hipSetDevice: *const fn (device: c_int) callconv(.c) c_int, - hipMemGetInfo: *const fn (free: *usize, total: *usize) callconv(.c) c_int, - hipMalloc: *const fn (ptr: *?*anyopaque, size: usize) callconv(.c) c_int, - hipFree: *const fn (ptr: ?*anyopaque) callconv(.c) c_int, - hipGetLastError: *const fn () callconv(.c) c_int, - - /// Only finds the library: it never loads it, nor unloads it (see the - /// note on thread-local storage in `load`). `libzml_rocm.so` needs - /// `libamdhip64.so.7`, and the plugin's own dlopen of `libamdhip64.so` - /// resolves to the same file (`zmlxrocm_dlopen`). - fn loaded() ?Hip { - var lib: std.DynLib = .{ .inner = .{ - .handle = std.c.dlopen("libamdhip64.so.7", .{ .LAZY = true, .NOLOAD = true }) orelse return null, - } }; - var hip: Hip = undefined; - inline for (@typeInfo(Hip).@"struct".field_names) |name| { - @field(hip, name) = lib.lookup(@FieldType(Hip, name), name) orelse return null; - } - return hip; - } - - /// Asks HIP for the whole memory of the device, made the current one, - /// when less than that is free, so that the driver cannot grant it. - /// Frees what HIP grants anyway. - fn requestWholeDevice(hip: Hip, ordinal: c_int) Answer { - const set_device = hip.hipSetDevice(ordinal); - if (set_device != hip_success) return .{ .set_device = set_device }; - var free: usize = 0; - var total: usize = 0; - const mem_get_info = hip.hipMemGetInfo(&free, &total); - if (mem_get_info != hip_success) return .{ .mem_get_info = mem_get_info }; - // ROCr rounds a request up to 2 MiB and refuses one beyond the - // device's size itself, without the driver's refusal that trims. - const request = std.mem.alignBackward(usize, total, 2 << 20); - if (free >= request) return .{ .not_asked = .{ .free = free, .request = request } }; - var ptr: ?*anyopaque = null; - const answer = hip.hipMalloc(&ptr, request); - if (answer == hip_error_out_of_memory) return .refused; - if (answer == hip_success) _ = hip.hipFree(ptr); - return .{ .malloc = answer }; - } -}; - -/// How a device answered `Hip.requestWholeDevice`. -const Answer = union(enum) { - /// hipMalloc answered `hipErrorOutOfMemory`, as expected. - refused, - /// So much is free that the driver could grant the request. - not_asked: struct { free: usize, request: usize }, - /// hipMalloc's other answers: `hip_success` for a grant, freed at once. - malloc: c_int, - /// The `hipError_t` of hipSetDevice or hipMemGetInfo, which failed. - set_device: c_int, - mem_get_info: c_int, - - pub fn format(answer: Answer, writer: *std.Io.Writer) std.Io.Writer.Error!void { - switch (answer) { - .refused => try writer.writeAll("refused"), - .not_asked => |memory| try writer.print("not asked, {d} bytes being free for a request of {d}", .{ memory.free, memory.request }), - .malloc => |code| if (code == hip_success) - try writer.writeAll("granted, then freed") - else - try writer.print("hipMalloc returned hipError_t {d}", .{code}), - .set_device => |code| try writer.print("hipSetDevice returned hipError_t {d}", .{code}), - .mem_get_info => |code| try writer.print("hipMemGetInfo returned hipError_t {d}", .{code}), - } - } -}; diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 4c1c50ed..6a0fdfae 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/stdx/Io.zig b/stdx/Io.zig index 453ee503..5b5a35d8 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -22,20 +22,130 @@ pub const Dir = struct { } }; +pub const LimitedGroup = struct { + limit: usize, + in_flight: std.atomic.Value(usize) = .init(0), + group: std.Io.Group = .init, + cond: std.Io.Condition = .init, + mutex: std.Io.Mutex = .init, + + fn Wrapper(comptime function: anytype) type { + return struct { + fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.limit) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } + + defer { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.cond.signal(io); + } + return @call(.auto, function, args); + } + + try self.mutex.lock(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.limit) { + try self.cond.wait(io, &self.mutex); + } + } + } + }; + } + + pub fn init(limit: usize) LimitedGroup { + return .{ .limit = limit }; + } + + pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { + self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { + try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *LimitedGroup, io: std.Io) void { + self.group.cancel(io); + } +}; + +pub const AllocatingLimitedConcurrentGroup = struct { + allocator: std.mem.Allocator, + + queue_buffer: []bool = &[_]bool{}, + limit: usize, + + group: std.Io.Group = .init, + queue: std.Io.Queue(bool) = undefined, + + pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { + const buffer = try allocator.alloc(bool, limit); + @memset(buffer, false); + var self: AllocatingLimitedConcurrentGroup = .{ + .allocator = allocator, + .queue_buffer = buffer, + .limit = limit, + .group = .init, + .queue = undefined, + }; + self.queue = std.Io.Queue(bool).init(self.queue_buffer); + return self; + } + + pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { + self.allocator.free(self.queue_buffer); + } + + pub fn concurrent( + self: *AllocatingLimitedConcurrentGroup, + io: std.Io, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + self.queue.putOneUncancelable(io, true) catch unreachable; + + try self.group.concurrent(io, struct { + fn wrapper( + io_: std.Io, + queue: *std.Io.Queue(bool), + inner_args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + defer _ = queue.getOneUncancelable(io_) catch unreachable; + return @call(.auto, function, inner_args); + } + }.wrapper, .{ io, &self.queue, args }); + } + + pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { + self.group.cancel(io); + } +}; + pub const Duration = struct { pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); } pub fn hz(self: std.Io.Duration) u64 { - return @intCast(@divTrunc(std.time.ns_per_s, self.nanoseconds)); + return @intCast(std.time.ns_per_s / self.nanoseconds); } pub fn hzFloat(self: std.Io.Duration) f64 { return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); } }; - -test "Duration.hz" { - try std.testing.expectEqual(@as(u64, 4), Duration.hz(.fromMilliseconds(250))); -} diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,25 +591,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 5ed0b048..1b19e638 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -87,33 +87,6 @@ pub fn Once(comptime f: anytype) type { }; } -/// The first error of concurrent tasks, kept once and readable by any of -/// them. The state is the error's integer value, zero while none is kept. -pub fn FirstError(comptime E: type) type { - return struct { - code: std.atomic.Value(@Int(.unsigned, @bitSizeOf(anyerror))) = .init(0), - - /// Keeps `err` unless an error is kept already; true when this call - /// kept it. - pub fn set(self: *@This(), err: E) bool { - return self.code.cmpxchgStrong(0, @intFromError(err), .release, .monotonic) == null; - } - - pub fn get(self: *const @This()) ?E { - const code = self.code.load(.acquire); - return if (code == 0) null else @errorCast(@errorFromInt(code)); - } - }; -} - -test FirstError { - var first: FirstError(error{ A, B }) = .{}; - try std.testing.expectEqual(null, first.get()); - try std.testing.expect(first.set(error.B)); - try std.testing.expect(!first.set(error.A)); - try std.testing.expectEqual(error.B, first.get().?); -} - /// Packed storage of sub-byte values /// /// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index d53e20be..69cc6128 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -30,7 +30,7 @@ zls_toolchain( toolchain( name = "toolchain", - target_settings = ["@zig_toolchains//:0.17.0"], + target_settings = ["@zig_toolchains//:any_0.17.release"], toolchain = ":zls", toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel new file mode 100644 index 00000000..aff4954d --- /dev/null +++ b/tools/safetensors/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "safetensors", + main = "main.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + "//stdx", + "//vfs", + "//zml", + ], +) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig new file mode 100644 index 00000000..cf902eef --- /dev/null +++ b/tools/safetensors/main.zig @@ -0,0 +1,91 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zml = @import("zml"); + +const log = std.log.scoped(.safetensors_convert); + +const Args = struct { + output: ?[]const u8 = null, + positional: struct { model: []const u8 }, + + pub const help = + \\Usage: [--output=] + \\ + \\Read safetensors headers and write model.safetensors.full_index.json in the working directory + \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash + ; +}; + +pub fn main(init: std.process.Init) !void { + const args = stdx.flags.parse(init.minimal.args, Args); + const allocator = init.gpa; + + const start = std.Io.Clock.now(.awake, init.io); + var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer client.deinit(); + try client.initDefaultProxies(allocator, init.environ_map); + + const input = args.positional.model; + + var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); + defer file_vfs.deinit(); + var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); + defer http_vfs.deinit(); + var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); + defer https_vfs.deinit(); + var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (hf_vfs) |*backend| backend.deinit(); + var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (s3_vfs) |*backend| backend.deinit(); + var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (gcs_vfs) |*backend| backend.deinit(); + + var vfs: VFS = try .init(allocator, init.io); + const io = vfs.io(); + defer vfs.deinit(); + try vfs.register("file", file_vfs.io()); + try vfs.register("http", http_vfs.io()); + try vfs.register("https", https_vfs.io()); + if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); + if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); + if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const repo = try zml.safetensors.resolveModelRepo(io, input); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const loaded = std.Io.Clock.now(.awake, init.io); + + const index_json_file = if (args.output) |output| + try std.Io.Dir.cwd().createFile(io, output, .{}) + else + try repo.createFile(io, full_index_path, .{}); + const out_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(out_buffer); + var out_writer = index_json_file.writer(io, out_buffer); + try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); + const done = std.Io.Clock.now(.awake, init.io); + + log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ + args.output orelse full_index_path, + registry.tensors.count(), + start.durationTo(loaded), + loaded.durationTo(done), + start.durationTo(done), + }); +} + +const full_index_path = "model.safetensors.full_index.json"; diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 352ce826..c8de1e0c 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -4,14 +4,11 @@ zig_library( name = "vfs", srcs = [ "base.zig", - "direct_io.zig", + "file.zig", "gcs.zig", "hf.zig", "http.zig", - "http_acceptance_test.zig", - "mock_server.zig", - "range_read.zig", - "request.zig", + "parallel_read.zig", "s3.zig", ], main = "vfs.zig", diff --git a/vfs/base.zig b/vfs/base.zig index fc21c0b7..1e66ca89 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -1,50 +1,9 @@ const std = @import("std"); -const builtin = @import("builtin"); const stdx = @import("stdx"); -/// `std.log.scoped`, except that test builds log `err` at `debug`: the test -/// runner counts logged errors as failures, and a test of a failure path -/// checks the error it returns. -pub fn scopedLog(comptime scope: @EnumLiteral()) type { - const scoped = std.log.scoped(scope); - return struct { - pub const debug = scoped.debug; - pub const info = scoped.info; - pub const warn = scoped.warn; - - pub fn err(comptime format: []const u8, args: anytype) void { - if (builtin.is_test) scoped.debug(format, args) else scoped.err(format, args); - } - }; -} - -/// How a backend's files are best read (`VFS.readHints`). No one source -/// request size suits every backend: warm local 8 MiB requests on one B70 -/// reached 27.05 GiB/s against 21.33 at 32 MiB, with a quarter of the pinned -/// memory; real AWS plateaued near 950 MiB/s, and 32 MiB was no faster. -pub const ReadHints = struct { - /// Local files, through a VFS or not. - pub const local: ReadHints = .{ .read_chunk_size = 8 * 1024 * 1024 }; - - /// Smallest source request expected to avoid excessive per-request cost - /// or backend rate limiting. - read_chunk_size: usize = 16 * 1024 * 1024, - /// The backend benefits from opening enough source calls to hide request - /// latency before the first response arrives. - high_latency: bool = false, -}; - -pub const Backend = struct { - io: std.Io, - read_hints: ReadHints = .{}, -}; - pub const VFSBase = struct { - /// Aligned like the i96 durations and timestamps a backend holds (its - /// `request.Governor`), so `@fieldParentPtr("base", ...)` needs no - /// `@alignCast`. - inner: std.Io align(@alignOf(std.Io.Duration)), + inner: std.Io, pub fn init(io: std.Io) VFSBase { return .{ .inner = io }; diff --git a/vfs/direct_io.zig b/vfs/direct_io.zig deleted file mode 100644 index 4d758589..00000000 --- a/vfs/direct_io.zig +++ /dev/null @@ -1,366 +0,0 @@ -//! Direct I/O on the descriptor of a local file. A VFS handle's `Mode` sets -//! the flag on its descriptor when a reader asks for it under a `Policy`, -//! and from then on issues that descriptor's positional reads here rather -//! than through the inner `std.Io`: the kernel answers an unsupported direct -//! read with `EINVAL` or `EFAULT`, which the threaded `std.Io` treats as a -//! programmer error, whereas here it is a refusal that clears the flag and -//! leaves the read to the VFS, buffered. Everything below `Mode` is Linux -//! only and reached from code gated on `supported`. -//! -//! Neither path wins everywhere, and residency alone cannot predict which -//! does: warm replicated Llama-3.1-8B on eight MI300X took ~1.10 s buffered -//! but ~4.53 s forced direct from a slow storage extent; on four GB300 with -//! four NVMe drives in RAID0, direct took ~0.30 s versus ~0.32 s warm -//! buffered. These are loader times, not disk-only rates. -const std = @import("std"); -const builtin = @import("builtin"); - -const linux = std.os.linux; - -const log = std.log.scoped(.@"zml/vfs"); - -pub const supported = builtin.target.os.tag == .linux and @hasField(std.posix.O, "DIRECT"); - -/// Alignment of a direct read's offset, every buffer and the total length. -/// 4 KiB covered the measured filesystems; this is not a statx(STATX_DIOALIGN) -/// query. A filesystem requiring more alignment may reject the first read, -/// after which the VFS demotes the handle to buffered. -pub const alignment: usize = 4 * 1024; - -/// `Policy.auto` reads a file directly when at most this fraction of its -/// sampled pages is cached. -const cold_fraction: f64 = 0.5; - -/// Whether a local file is read past the page cache. A direct read comes -/// from the disk even when the cache holds the data; it also leaves the cache -/// as it found it, so a file read directly is still cold for the next load -/// (the `auto` probe caches what it samples, elsewhere on every probe: -/// `cachedFraction`). -/// Raw warm-cache reads exceeded disk throughput in the measured setups, -/// but the loader can still favor direct reads: page-cache copies compete -/// with DMA for host memory bandwidth. Conversely, repeated loads from a -/// slow disk can benefit greatly from warming the cache with `off`. The -/// contrasting GB300/MI300X load results are in this file's header. -pub const Policy = enum { - off, - /// Every file the filesystem allows. - on, - /// A file mostly out of the page cache when asked, buffered for a cached - /// one. Residency is sampled: `residency_samples` pages spread over the - /// file, read with `RWF_NOWAIT`, which answers `EAGAIN` for a page that - /// is not cached, needs no permission on the file and costs the same - /// whatever its size. Unknown counts as cached. - auto, -}; - -/// A local handle's read mode: undecided until a reader asks for direct -/// reads (`VFS.useDirectIo`), then buffered or direct for the rest of the -/// handle's life, since the flag belongs to the open file description and a -/// reader that was answered buffered plans exact reads that a direct -/// descriptor would reject. A direct handle goes back to buffered once the -/// kernel rejected a read, or a read went through the inner `Io`. Each -/// method takes the inner `Io` and the handle's file on it. -pub const Mode = struct { - /// Changes only under `mutex`, together with the flag; reads only load it. - state: std.atomic.Value(State) = .init(.undecided), - mutex: std.Io.Mutex = .init, - - const State = enum(u8) { undecided, buffered, direct }; - - /// Whether `file` reads direct, deciding an undecided handle under - /// `policy` (`VFS.useDirectIo`); `off` decides nothing. - pub fn decide(self: *Mode, io: std.Io, file: std.Io.File, policy: Policy) bool { - if (comptime !supported) return false; - if (policy == .off) return false; - switch (self.state.load(.acquire)) { - .direct => return true, - .buffered => return false, - .undecided => {}, - } - const stat = file.stat(io) catch return self.settle(io, file, false); - if (!fits(stat)) return self.settle(io, file, false); - if (policy == .auto) { - const name: Name = .{ .io = io, .file = file }; - const cached = sampleResidency(io, file, stat.size) orelse { - log.debug("{f}: page cache residency unknown; reading buffered", .{name}); - return self.settle(io, file, false); - }; - if (cached > cold_fraction) { - log.debug("{f}: {d:.0}% cached; reading buffered", .{ name, cached * 100 }); - return self.settle(io, file, false); - } - log.debug("{f}: {d:.0}% cached", .{ name, cached * 100 }); - } - return self.settle(io, file, true); - } - - /// Decides an undecided handle under the mutex, setting the flag first - /// when `direct`, and returns whether the handle reads direct: a - /// decision already taken (by a racing call, or a demotion) stands. A - /// descriptor the flag cannot go on (`enable`) is read buffered. - fn settle(self: *Mode, io: std.Io, file: std.Io.File, direct: bool) bool { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - switch (self.state.load(.acquire)) { - .direct => return true, - .buffered => return false, - .undecided => {}, - } - if (direct) { - const name: Name = .{ .io = io, .file = file }; - const err = enable(file.handle); - if (err == .SUCCESS) { - self.state.store(.direct, .release); - log.debug("{f}: reading direct", .{name}); - return true; - } - log.debug("{f}: O_DIRECT refused ({t}); reading buffered", .{ name, err }); - } - self.state.store(.buffered, .release); - return false; - } - - /// The bytes read from `offset` into `data` past the page cache, or - /// null for the caller to read them buffered: the handle is not direct, - /// or the kernel refused the read and the handle is buffered from now - /// on (`logRefusal`). A refusal demotes rather than fails: a legitimate - /// short direct read can leave an unaligned continuation, and a - /// filesystem accepting the flag does not prove it accepts these - /// buffers (including PJRT-pinned memory) at read time. - pub fn read(self: *Mode, io: std.Io, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!?usize { - if (comptime !supported) return null; - if (self.state.load(.acquire) != .direct) return null; - switch (try readPositional(file.handle, data, offset)) { - .bytes => |bytes| return bytes, - .refused => |err| { - if (self.demote(io, file)) logRefusal(.{ .io = io, .file = file }, err, data, offset); - return null; - }, - } - } - - /// Takes a direct handle back to buffered before a read the VFS does - /// not issue itself (streaming, or as the source of a file copy): the - /// inner `Io` treats the errno of a rejected direct read as a - /// programmer error. - pub fn leave(self: *Mode, io: std.Io, file: std.Io.File) void { - if (comptime !supported) return; - if (self.state.load(.acquire) != .direct or !self.demote(io, file)) return; - const name: Name = .{ .io = io, .file = file }; - log.warn("{f}: buffered read of a direct file; reading it buffered from now on", .{name}); - } - - /// Takes a direct handle back to buffered: the flag comes off the - /// descriptor before the state says buffered, under the mutex, so the - /// inner `Io` never reads a descriptor with the flag on. True for the - /// call that did it, which then says why. - fn demote(self: *Mode, io: std.Io, file: std.Io.File) bool { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - if (self.state.load(.acquire) != .direct) return false; - disable(file.handle); - self.state.store(.buffered, .release); - return true; - } -}; - -/// A file's size, and whether `Mode.decide` would read it direct -/// (`predict`). -pub const Prediction = struct { size: u64, direct: bool }; - -/// `Mode.decide`'s answer for `file` under `policy`, deciding and logging -/// nothing: the same rule, then the flag set on `file` and left set, so -/// `file` must be a read-only descriptor of the caller's own, which the -/// caller closes. Under `auto` the sample caches what it misses, as -/// `decide`'s does: a missed sample queues the read of its page and of the -/// kernel's readahead. A decision that follows and samples those pages -/// once the reads have landed finds the file cached and reads it buffered, -/// which may happen to any file, the likelier the fewer pages it has per -/// sample (`cachedFraction`). Null when `file` cannot be stat-ed. -pub fn predict(io: std.Io, file: std.Io.File, policy: Policy) error{Canceled}!?Prediction { - if (comptime !supported) return null; - const stat = file.stat(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => return null, - }; - const direct = fits(stat) and switch (policy) { - .off => false, - .on => true, - .auto => if (sampleResidency(io, file, stat.size)) |cached| cached <= cold_fraction else false, - }; - return .{ .size = stat.size, .direct = direct and enable(file.handle) == .SUCCESS }; -} - -/// A regular file at least one unit long: the only kind read direct -/// (`Mode.decide`, `predict`). -fn fits(stat: std.Io.File.Stat) bool { - return stat.kind == .file and stat.size >= alignment; -} - -/// A local file's path in a log line, resolved only when the line is -/// written. -const Name = struct { - io: std.Io, - file: std.Io.File, - - pub fn format(self: Name, writer: *std.Io.Writer) !void { - var buffer: [std.fs.max_path_bytes]u8 = undefined; - const len = self.file.realPath(self.io, &buffer) catch return writer.writeAll(""); - try writer.writeAll(buffer[0..len]); - } -}; - -/// The kernel rejected a direct read. An aligned one is the filesystem -/// refusing direct I/O it accepted the flag for, a warning; a misaligned -/// one is the reader's, either its plan or a continuation after a short -/// read (which a network filesystem may answer), an error. Either way the -/// file is read buffered from now on. -fn logRefusal(name: Name, err: linux.E, data: []const []u8, offset: u64) void { - var total: usize = 0; - var aligned = offset % alignment == 0; - for (data) |buffer| { - if (buffer.len == 0) continue; - aligned = aligned and @intFromPtr(buffer.ptr) % alignment == 0; - total += buffer.len; - } - aligned = aligned and total % alignment == 0; - if (aligned) { - log.warn("{f}: direct read refused ({t}) at offset={d} total={d}; reading the file buffered from now on", .{ name, err, offset, total }); - } else { - log.err("{f}: misaligned read of a direct file ({t}) at offset={d} first_buffer=0x{x} total={d} alignment={d}; reading the file buffered from now on", .{ - name, - err, - offset, - if (data.len != 0) @intFromPtr(data[0].ptr) else 0, - total, - alignment, - }); - } -} - -const max_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; -const direct_flag: usize = @as(u32, @bitCast(std.posix.O{ .DIRECT = true })); - -/// Sets the flag on a descriptor opened for reading only, since the flag -/// constrains writes too: one opened for writing is refused with `BADF`. A -/// filesystem without direct I/O refuses the flag. -pub fn enable(fd: std.posix.fd_t) linux.E { - const current = switch (getStatusFlags(fd)) { - .flags => |flags| flags, - .err => |err| return err, - }; - const status: std.posix.O = @bitCast(@as(u32, @truncate(current))); - if (status.ACCMODE != .RDONLY) return .BADF; - return setStatusFlags(fd, current | direct_flag); -} - -/// Clears the flag, so the inner `Io` may read the descriptor again. -fn disable(fd: std.posix.fd_t) void { - const current = switch (getStatusFlags(fd)) { - .flags => |flags| flags, - .err => return, - }; - _ = setStatusFlags(fd, current & ~direct_flag); -} - -/// `fcntl(F_GETFL)`, retried on `EINTR`. -fn getStatusFlags(fd: std.posix.fd_t) union(enum) { flags: usize, err: linux.E } { - while (true) { - const rc = linux.fcntl(fd, linux.F.GETFL, 0); - switch (linux.errno(rc)) { - .SUCCESS => return .{ .flags = rc }, - .INTR => continue, - else => |err| return .{ .err = err }, - } - } -} - -/// `fcntl(F_SETFL)`, retried on `EINTR`. -fn setStatusFlags(fd: std.posix.fd_t, flags: usize) linux.E { - while (true) { - switch (linux.errno(linux.fcntl(fd, linux.F.SETFL, flags))) { - .INTR => continue, - else => |err| return err, - } - } -} - -/// A fixed count bounds the probe's cost whatever the checkpoint's size and -/// suffices for the coarse half-cached decision. A whole-file `cachestat` -/// walks every cached folio, about 7 ms per cached GiB on a 4 KiB-page -/// kernel, paid on the planner thread, and refuses a file the process can -/// read but not write. -pub const residency_samples = 32; - -/// The fraction of `residency_samples` pages spread over the file that the -/// page cache holds, or null when the filesystem cannot say (no -/// `RWF_NOWAIT`). One sample per window of the file, each `shift` bytes -/// into its window (`shift` modulo the window), unknown residency selecting -/// buffered IO. A sample that misses queues the read of its page, which a -/// direct file never reads back: with the same offsets the next probe would -/// find every sample cached and read a cold file buffered, so callers draw a -/// new `shift` for every probe. A file of at most `residency_samples` pages -/// has every page sampled whatever the shift, so its first probe caches it. -/// The header page a parser has just read is sampled only when `shift` -/// lands in it, not systematically. -fn cachedFraction(fd: std.posix.fd_t, size: u64, shift: u64) ?f64 { - var buffer: [512]u8 = undefined; - const iovec: std.posix.iovec = .{ .base = &buffer, .len = buffer.len }; - var cached: usize = 0; - const window = size / residency_samples; - for (0..residency_samples) |sample| { - const offset = window * sample + shift % @max(window, 1); - const rc = linux.preadv2(fd, @ptrCast(&iovec), 1, @intCast(offset), linux.RWF.NOWAIT); - switch (linux.errno(rc)) { - .SUCCESS => cached += @intFromBool(rc != 0), - .AGAIN => {}, - else => return null, - } - } - return @as(f64, @floatFromInt(cached)) / @as(f64, @floatFromInt(residency_samples)); -} - -/// `cachedFraction` at a fresh shift, as every probe draws one. -fn sampleResidency(io: std.Io, file: std.Io.File, size: u64) ?f64 { - var shift: u64 = undefined; - io.random(std.mem.asBytes(&shift)); - return cachedFraction(file.handle, size, shift); -} - -const ReadResult = union(enum) { - bytes: usize, - /// The kernel rejected the direct read: an unaligned offset, buffer or - /// total, a filesystem that took the flag but not the read, or memory - /// the block layer cannot pin. - refused: linux.E, -}; - -/// Reads `data` at `offset` on a descriptor with the flag set, at most -/// `IOV_MAX` buffers per call (the caller loops). A read that runs past the -/// end of the file is cut there. -fn readPositional(fd: std.posix.fd_t, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!ReadResult { - var iovecs: [max_iovecs]std.posix.iovec = undefined; - var count: usize = 0; - for (data) |buffer| { - if (buffer.len == 0) continue; - if (count == iovecs.len) break; - iovecs[count] = .{ .base = buffer.ptr, .len = buffer.len }; - count += 1; - } - if (count == 0) return .{ .bytes = 0 }; - while (true) { - const rc = linux.preadv(fd, &iovecs, count, @intCast(offset)); - switch (linux.errno(rc)) { - .SUCCESS => return .{ .bytes = rc }, - .INTR => continue, - .INVAL, .FAULT, .OPNOTSUPP => |err| return .{ .refused = err }, - .IO => return error.InputOutput, - .NOMEM => return error.SystemResources, - .AGAIN => return error.WouldBlock, - .ISDIR => return error.IsDir, - .BADF => return error.NotOpenForReading, - .NXIO, .SPIPE => return error.Unseekable, - else => |err| return std.posix.unexpectedErrno(err), - } - } -} diff --git a/vfs/file.zig b/vfs/file.zig new file mode 100644 index 00000000..1b077c33 --- /dev/null +++ b/vfs/file.zig @@ -0,0 +1,430 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/file"); + +pub const DirectIoError = error{ + UnsupportedPlatform, + MisalignedBuffer, + BufferTooSmall, + UnexpectedFcntlResult, +}; + +fn canUseDirectIO() bool { + if (builtin.target.os.tag == .linux) { + return @hasField(std.posix.O, "DIRECT"); + } + return false; +} + +fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { + while (true) { + const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); + switch (std.posix.errno(result)) { + .SUCCESS => return result, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { + while (true) { + switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { + .SUCCESS => return, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn useDirectIO(file: std.Io.File) DirectIoError!bool { + if (comptime canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + return (flags & direct_flag) != 0; + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { + if (comptime canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + if ((flags & direct_flag) == 0) return; + + try setStatusFlags(file, flags & ~direct_flag); + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToDirectIO(file: std.Io.File) DirectIoError!void { + if (builtin.os.tag == .linux and canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + try setStatusFlags(file, flags | direct_flag); + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +pub const File = struct { + pub const Config = struct { + direct_io: bool = true, + direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), + }; + + const Handle = struct { + inner_handle: std.Io.File.Handle, + inner_flags: std.Io.File.Flags, + direct_io_handle: ?std.Io.File.Handle, + + pub fn innerFile(self: *Handle) std.Io.File { + return .{ + .handle = self.inner_handle, + .flags = self.inner_flags, + }; + } + + pub fn directIoFile(self: *Handle) std.Io.File { + return .{ + .handle = self.direct_io_handle.?, + .flags = self.inner_flags, + }; + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + config: Config, + base: VFSBase, + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { + return .{ + .allocator = allocator, + .config = config, + .base = .init(inner), + }; + } + + pub fn deinit(self: *File) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + } + + pub fn io(self: *File) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *File) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *File, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *File, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.handles.at(@intCast(file.handle)); + } + + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { + const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { + else => return std.Io.File.OpenError.Unexpected, + }; + + return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); + } + + fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { + const handle = self.getFileHandle(file); + + if (handle.direct_io_handle == null) { + return handle.innerFile(); + } + + const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); + + if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { + return handle.innerFile(); + } + + var total_size: usize = 0; + for (data) |buf| { + if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { + return handle.innerFile(); + } + total_size += buf.len; + } + + if (!std.mem.isAligned(total_size, alignment_bytes)) { + return handle.innerFile(); + } + + return handle.directIoFile(); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); + const inner_file = self.innerFile(o.file, o.data, position); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_read_streaming = .{ + .file = inner_file, + .data = o.data, + }, + }); + }, + .file_write_streaming => |o| { + const handle = self.getFileHandle(o.file); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_write_streaming = .{ + .file = handle.innerFile(), + .header = o.header, + .data = o.data, + .splat = o.splat, + }, + }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = null, + }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); + + var direct_io_handle: ?std.Io.File.Handle = null; + if (use_direct_io) { + const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); + + switchToDirectIO(direct_io_file) catch |err| switch (err) { + DirectIoError.UnsupportedPlatform => { + log.debug("Direct I/O is not supported on this platform", .{}); + }, + else => { + log.err("Failed to switch to Direct I/O mode: {any}", .{err}); + return std.Io.File.OpenError.Unexpected; + }, + }; + + if (useDirectIO(direct_io_file) catch false) { + direct_io_handle = direct_io_file.handle; + } + } + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = direct_io_handle, + }; + + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle = self.getFileHandle(file); + + if (handle.direct_io_handle) |_| { + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); + } + + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); + + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = self.innerFile(file, data, offset); + return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); + } +}; diff --git a/vfs/gcs.zig b/vfs/gcs.zig index 6717c59b..acc7f634 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -3,15 +3,13 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const range_read = @import("range_read.zig"); -const request = @import("request.zig"); +const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/gcs"); +const EndpointUrl = "https://storage.googleapis.com"; const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; -const TokenUrl = "https://oauth2.googleapis.com/token"; /// Check if running on Google Compute Engine / Cloud Run host infrastructure, /// because metadata server probing should only happen on GCP. @@ -53,11 +51,29 @@ const Credentials = union(enum) { }; const ServiceAccount = struct { + pub const JwtAssertion = struct { + iss: []const u8, + scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", + aud: []const u8 = "https://oauth2.googleapis.com/token", + exp: i64, + iat: i64, + sub: ?[]const u8 = null, + }; + type: []const u8, client_email: []const u8, private_key: []const u8, token_uri: []const u8, quota_project_id: ?[]const u8 = null, + + pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { + const now: std.Io.Timestamp = .now(io_, .real); + return .{ + .iss = self.client_email, + .iat = now.toSeconds(), + .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), + }; + } }; authorized_user: AuthorizedUser, @@ -106,36 +122,159 @@ const Credentials = union(enum) { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const GCS = struct { - const authorization_header_size = "Bearer ".len + OAuthToken.Size; + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *GCS, + uri: std.Uri, + path: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request( + .GET, + job.batch.uri, + .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }, + ) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; const Token = struct { - /// Guards the token. A refresh is a governed request that may wait - /// out a hold or retry for minutes, so it runs under this lock, not - /// under `GCS.mutex`, which every handle lookup takes. - mutex: std.Io.Mutex = .init, - header_storage: []u8, - header_len: usize = 0, + header: []u8, expires_at: std.Io.Timestamp, - /// Refreshes that failed other than by a cancel. A reader that - /// queued behind one fails with it instead of running its own. - failures: std.atomic.Value(u64) = .init(0), fn expired(self: *const Token, io_: std.Io) bool { const now: std.Io.Timestamp = .now(io_, .real); return now.toSeconds() >= self.expires_at.toSeconds(); } - - fn header(self: *const Token) []const u8 { - return self.header_storage[0..self.header_len]; - } }; pub const Config = struct { credentials: ?Credentials = null, endpoint_uri: std.Uri, - /// Where tokens come from; a test points them at a local server. - metadata_uri: std.Uri = std.Uri.parse(MetadataUrl) catch unreachable, - token_uri: std.Uri = std.Uri.parse(TokenUrl) catch unreachable, + region: []const u8 = "auto", }; const Handle = struct { @@ -163,33 +302,49 @@ pub const GCS = struct { allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, mutex: std.Io.Mutex = .init, + client: *std.http.Client, config: Config, token: Token, - governor: request.Governor, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub const Options = struct { + pub const InitOpts = struct { credentials: ?union(enum) { json: *std.Io.Reader, metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", - retry: request.Retry = .{}, + region: []const u8 = "auto", + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, }; pub const InitError = error{ InvalidCredentialJson, Unexpected, - } || std.mem.Allocator.Error; + } || std.mem.Allocator.Error || std.Io.ConcurrentError; - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) InitError!GCS { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); + + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + const config: Config = .{ - .credentials = if (options.credentials) |creds| switch (creds) { + .credentials = if (opts.credentials) |creds| switch (creds) { .json => |reader| blk: { var json_reader: std.json.Reader = .init(allocator, reader); defer json_reader.deinit(); @@ -203,11 +358,12 @@ pub const GCS = struct { }, .metadata_server => .{ .metadata_server = {} }, } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, options.endpoint_url)) catch return InitError.Unexpected, + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, + .region = try arena.allocator().dupe(u8, opts.region), }; const token: Token = .{ - .header_storage = try arena.allocator().alloc(u8, authorization_header_size), + .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), .expires_at = .zero, }; @@ -215,10 +371,10 @@ pub const GCS = struct { .allocator = allocator, .arena = arena, .base = .init(inner), + .client = http_client, .config = config, .token = token, - // GCS answers rate limiting with `503` as well as `429`. - .governor = .init(inner, http_client, "gcs", .throttle, options.retry), + .read_pool = read_pool, }; } @@ -263,70 +419,49 @@ pub const GCS = struct { return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; } - /// The metadata server's token, through the governed loop: a throttled - /// token endpoint holds the backend like any other request. - fn refreshMetadataServerToken(self: *GCS, buffer: []u8) ![]const u8 { - var token: TokenRequest = .{ - .uri = self.config.metadata_uri, - .target = "the metadata server", - .buffer = buffer, - .extra_headers = &.{.{ .name = "Metadata-Flavor", .value = "Google" }}, - }; - return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); - } - - fn refreshAuthorizedUserToken(self: *GCS, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { - // Its own buffer: a retried attempt sends it again after the - // response was written into `buffer`. - var payload_buffer: [2 * 1024]u8 = undefined; - const payload = try std.fmt.bufPrint(&payload_buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = MetadataUrl }, + .method = .GET, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + }, + .extra_headers = &.{ + .{ .name = "Metadata-Flavor", .value = "Google" }, + }, + .response_writer = &response_writer, }); - var token: TokenRequest = .{ - .uri = self.config.token_uri, - .target = "the ADC token endpoint", - .buffer = buffer, - .method = .POST, - .payload = payload, - .content_type = .{ .override = "application/x-www-form-urlencoded" }, - }; - return request.perform([]const u8, &self.governor, token.spec(), &token, TokenRequest.attempt); + + if (result.status != .ok) { + return null; + } + + return response_writer.buffered(); } - /// One OAuth token exchange, whose body it copies into `buffer`. - const TokenRequest = struct { - uri: std.Uri, - target: []const u8, - buffer: []u8, - method: std.http.Method = .GET, - payload: ?[]u8 = null, - content_type: std.http.Client.Request.Headers.Value = .default, - extra_headers: []const std.http.Header = &.{}, - - fn spec(self: *const TokenRequest) request.RequestSpec { - return .{ .target = self.target, .key = request.authorityOf(self.uri) }; + fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = "https://oauth2.googleapis.com/token" }, + .method = .POST, + .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }), + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }, + .response_writer = &response_writer, + }); + if (result.status != .ok) { + log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + return error.RequestFailed; } - - fn attempt(self: *TokenRequest, current: request.Attempt) anyerror!request.Outcome([]const u8) { - return request.exchange([]const u8, current, self.uri, .{ - .method = self.method, - .content_type = self.content_type, - .extra_headers = self.extra_headers, - .payload = self.payload, - .redirects = .follow, - // A refused exchange's body says why (`invalid_grant`, ...). - .log_refusal_body = true, - }, self, TokenRequest.consume); - } - - fn consume(self: *TokenRequest, res: *std.http.Client.Response) anyerror![]const u8 { - var writer: std.Io.Writer = .fixed(self.buffer); - _ = try res.reader(&.{}).streamRemaining(&writer); - return writer.buffered(); - } - }; + return response_writer.buffered(); + } fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { _ = io_; @@ -334,49 +469,79 @@ pub const GCS = struct { _ = service_account; _ = buffer; return error.Unimplemented; + + // const assertion = service_account.makeAssertion(io_); + + // var tmp_buffer: [8 * 1024]u8 = undefined; + + // var buffer_writer: std.Io.Writer = .fixed(buffer); + // var signing_input_buffer: [2 * 1024]u8 = undefined; + // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); + + // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; + // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); + // std.base64.url_safe_no_pad.Encoder.encodeWriter( + // signing_input_writer, + // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ + // std.json.fmt(assertion, .{}), + // }), + // ); + // const signing_input = signing_input_writer.buffered(); + // _ = signing_input; + + // const result = try client.fetch(.{ + // .location = .{ .url = "https://oauth2.googleapis.com/token" }, + // .method = .POST, + // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + // }), + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .content_type = .{ .override = "application/x-www-form-urlencoded" }, + // }, + // .response_writer = &response_writer, + // }); + // if (result.status != .ok) { + // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + // return error.RequestFailed; + // } } fn refreshToken(self: *GCS) !void { var buffer: [4 * 1024]u8 = undefined; const payload = switch (self.config.credentials.?) { - .authorized_user => |authorized_user| try self.refreshAuthorizedUserToken(authorized_user, &buffer), - .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.governor.client, service_account, &buffer), - .metadata_server => try self.refreshMetadataServerToken(&buffer), + .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), + .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, }; const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ .allocate = .alloc_if_needed, .ignore_unknown_fields = true, }); defer oauth_token.deinit(); - const header = try std.fmt.bufPrint(self.token.header_storage, "Bearer {s}", .{oauth_token.value.access_token}); - self.token.header_len = header.len; - self.token.expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))); + self.token = .{ + .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), + .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), + }; } - fn getOrRefreshToken(self: *GCS, authorization_buffer: []u8) !std.http.Client.Request.Headers.Value { + fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { if (self.config.credentials == null) { return .omit; } - const failures = self.token.failures.load(.monotonic); - try self.token.mutex.lock(self.base.inner); - defer self.token.mutex.unlock(self.base.inner); if (self.token.expired(self.base.inner)) { - // The refresh this reader waited behind failed, and its reader - // logged why: repeating it, with a retry budget of its own, would - // only keep every other queued reader waiting. - if (self.token.failures.load(.monotonic) != failures) return error.RequestFailed; - self.refreshToken() catch |err| { - if (err != error.Canceled) _ = self.token.failures.fetchAdd(1, .monotonic); - return err; - }; + try self.refreshToken(); } - // A copy: a refresh by another reader rewrites the token's storage. - return .{ .override = try std.fmt.bufPrint(authorization_buffer, "{s}", .{self.token.header()}) }; + return .{ .override = self.token.header }; } pub fn deinit(self: *GCS) void { - self.governor.logTotals(); + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -426,13 +591,6 @@ pub const GCS = struct { }; } - pub fn backend(self: *GCS) Backend { - return .{ - .io = self.io(), - .read_hints = .{ .high_latency = true }, - }; - } - fn openHandle(self: *GCS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -482,9 +640,9 @@ pub const GCS = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { - error.Canceled => return error.Canceled, - error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { @@ -530,9 +688,11 @@ pub const GCS = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; return .{ .inode = 0, @@ -547,19 +707,18 @@ pub const GCS = struct { }; } - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; - _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); - } + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -585,16 +744,7 @@ pub const GCS = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch |err| switch (err) { - error.Canceled => return error.Canceled, - error.OutOfMemory => return error.SystemResources, - // Logged by `exchange`. - error.RequestFailed => return error.Unexpected, - else => { - log.err("listing {s} failed: {t}", .{ handle.uri, err }); - return error.Unexpected; - }, - }; + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -679,7 +829,10 @@ pub const GCS = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -689,7 +842,7 @@ pub const GCS = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @abs(relative_offset); + const abs_offset: u64 = @intCast(-relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -711,6 +864,25 @@ pub const GCS = struct { return path.len; } + fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @backingInt(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { const path = std.mem.trim(u8, path_, "/"); @@ -721,16 +893,43 @@ pub const GCS = struct { } } - /// The object's URI. A VFS path is `bucket/key`; the request target - /// needs `/bucket/key`, formatted into `buffer`, which the URI borrows. - fn gcsUri(self: *GCS, path: []const u8, buffer: []u8) !std.Uri { + fn gcsUri(self: *GCS, path: []const u8) std.Uri { var uri = self.config.endpoint_uri; - const absolute = if (std.mem.startsWith(u8, path, "/")) path else std.fmt.bufPrint(buffer, "/{s}", .{path}) catch return error.NameTooLong; - uri.path = .{ .raw = absolute }; + uri.path = .{ .raw = path }; uri.query = .{ .percent_encoded = "alt=media" }; return uri; } + // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { + // authorization: ?[]const u8, + // extra_headers: [3]?std.http.Header, + // extra_len: usize, + // } { + // if (self.config.oauth_access_token == null) { + // if (self.config.googleCredentialsJsonStr) |json| { + // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); + // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); + // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); + // self.config.oauth_access_token = oauth.access_token; + // self.config.quota_project_id = oauth.quota_project_id; + // } + // } + + // const token = self.config.oauth_access_token orelse return error.MissingCredentials; + // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); + // var extra_headers: [3]?std.http.Header = .{ null, null, null }; + // var extra_len: usize = 0; + // if (self.config.quota_project_id) |project| { + // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; + // extra_len = 1; + // } + // return .{ + // .authorization = authorization, + // .extra_headers = extra_headers, + // .extra_len = extra_len, + // }; + // } + fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -752,35 +951,35 @@ pub const GCS = struct { uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; uri.query = .{ .percent_encoded = query_writer.buffered() }; - var listing: Listing = .{ .gcs = self, .uri = uri }; - return request.perform([]u8, &self.governor, .{ .target = bucket, .key = request.authorityOf(uri) }, &listing, Listing.attempt); - } - - /// The bucket listing. The bearer token is copied per attempt, so one - /// that expired during a hold is refreshed before the next try. - const Listing = struct { - gcs: *GCS, - uri: std.Uri, - authorization: [authorization_header_size]u8 = undefined, - - fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { - return request.exchange([]u8, current, self.uri, .{ - .authorization = try self.gcs.getOrRefreshToken(&self.authorization), - .extra_headers = if (self.gcs.quotaProjectId()) |project| - &.{.{ .name = "x-goog-user-project", .value = project }} - else - &.{}, - }, self, Listing.consume); - } - - fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { - const allocator = self.gcs.allocator; - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(allocator, content_len) + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = if (self.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} else - try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); + &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; } - }; + + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { const bucket, const key_prefix = self.pathComponents(prefix); @@ -831,65 +1030,76 @@ pub const GCS = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *GCS, path: []const u8) !u64 { - var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; - const uri = try self.gcsUri(path, &uri_path_buffer); - var head: SizeRequest = .{ .gcs = self, .uri = uri }; - return request.perform(u64, &self.governor, .{ .target = path, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); + fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const uri = self.gcsUri(path); + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + const size = switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + .unauthorized, .forbidden => return error.PermissionDenied, + else => blk: { + log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + return size; } - /// The object's size: one HEAD. An absent or denied object fails the - /// request (`ExchangeOptions.open`). - const SizeRequest = struct { - gcs: *GCS, - uri: std.Uri, - authorization: [authorization_header_size]u8 = undefined, - - fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { - return request.exchange(u64, current, self.uri, .{ - .method = .HEAD, - .authorization = try self.gcs.getOrRefreshToken(&self.authorization), - .open = true, - }, self, SizeRequest.consume); - } - - fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { - return res.head.content_length orelse error.MissingContentLength; - } - }; - fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - var uri_path_buffer: [8 * 1024 + 1]u8 = undefined; - var bearer: BearerRequest = .{ .gcs = self, .uri = try self.gcsUri(handle.uri, &uri_path_buffer) }; - return range_read.performRangeRead(&self.governor, .{ - .request = .{ .target = handle.uri, .key = request.authorityOf(bearer.uri) }, - }, &bearer, BearerRequest.prepare, data, offset, handle.size); - } + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; - /// The bearer token is copied per attempt, so one that expired during a - /// retry delay is refreshed before the next GET. - const BearerRequest = struct { - gcs: *GCS, - uri: std.Uri, - authorization: [authorization_header_size]u8 = undefined, + const uri = self.gcsUri(handle.uri); + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); - fn prepare(self: *BearerRequest, _: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { - return .{ .uri = self.uri, .authorization = try self.gcs.getOrRefreshToken(&self.authorization) }; + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .path = handle.uri, + .authorization = try self.getOrRefreshToken(), + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - }; + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } }; -test "GCS object URIs are rooted at the endpoint" { - var backend: GCS = undefined; - backend.config = .{ .endpoint_uri = try std.Uri.parse("https://storage.googleapis.com") }; - - var path_buffer: [64]u8 = undefined; - const uri = try backend.gcsUri("bucket/dir/object.bin", &path_buffer); - var uri_buffer: [128]u8 = undefined; - const formatted = try std.fmt.bufPrint(&uri_buffer, "{f}", .{uri}); - try std.testing.expectEqualStrings("https://storage.googleapis.com/bucket/dir/object.bin?alt=media", formatted); -} - test "GCS parses XML bucket listing objects and common prefixes" { const xml = "" ++ @@ -926,272 +1136,3 @@ test "GCS splits bucket and object prefix" { try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); try std.testing.expectEqualStrings("LC08/01", prefix); } - -/// A token endpoint at `/token` and one object at `/bucket/object`, for the -/// token tests. The nth token request gets `answers[n]`, the last repeating; -/// a 200 grants `tok-`, k counting the grants. -const TokenServer = struct { - tcp: std.Io.net.Server, - group: std.Io.Group = .init, - object: []const u8, - answers: []const std.http.Status, - /// How long each token answer takes, so that readers queue behind a - /// refresh. - delay: std.Io.Duration = .fromMilliseconds(100), - /// The body every token request must POST form-encoded, if any. - form: ?[]const u8 = null, - token_requests: std.atomic.Value(usize) = .init(0), - granted: std.atomic.Value(usize) = .init(0), - object_requests: std.atomic.Value(usize) = .init(0), - /// Object requests without the last token granted. - stale_requests: std.atomic.Value(usize) = .init(0), - /// Token requests that did not POST `form`. - bad_forms: std.atomic.Value(usize) = .init(0), - first_error: stdx.FirstError(anyerror) = .{}, - - fn init(io: std.Io, object: []const u8, answers: []const std.http.Status) !TokenServer { - const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; - return .{ .tcp = try address.listen(io, .{ .reuse_address = true }), .object = object, .answers = answers }; - } - - fn deinit(self: *TokenServer, io: std.Io) void { - self.group.cancel(io); - self.tcp.deinit(io); - } - - fn run(self: *TokenServer, io: std.Io) std.Io.Cancelable!void { - var connections: std.Io.Group = .init; - defer connections.cancel(io); - while (true) { - const stream = self.tcp.accept(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => { - _ = self.first_error.set(err); - return; - }, - }; - connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { - stream.close(io); - _ = self.first_error.set(err); - return; - }; - } - } - - fn onConnection(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) std.Io.Cancelable!void { - defer stream.close(io); - self.answer(stream, io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => _ = self.first_error.set(err), - }; - } - - fn answer(self: *TokenServer, stream: std.Io.net.Stream, io: std.Io) !void { - var read_buffer: [8 * 1024]u8 = undefined; - var reader = stream.reader(io, &read_buffer); - var write_buffer: [8 * 1024]u8 = undefined; - var writer = stream.writer(io, &write_buffer); - var server: std.http.Server = .init(&reader.interface, &writer.interface); - var req = try server.receiveHead(); - - if (std.mem.eql(u8, req.head.target, "/token")) { - const n = self.token_requests.fetchAdd(1, .acq_rel); - if (self.form) |form| { - const content_type = req.head.content_type orelse ""; - const form_encoded = req.head.method == .POST and std.mem.eql(u8, content_type, "application/x-www-form-urlencoded"); - var body_buffer: [512]u8 = undefined; - var body: std.Io.Writer = .fixed(&body_buffer); - _ = try req.readerExpectNone(&.{}).streamRemaining(&body); - if (!form_encoded or !std.mem.eql(u8, body.buffered(), form)) _ = self.bad_forms.fetchAdd(1, .acq_rel); - } - try io.sleep(self.delay, .awake); - const status = self.answers[@min(n, self.answers.len - 1)]; - if (status != .ok) return req.respond("{\"error\":\"invalid_grant\"}", .{ .status = status, .keep_alive = false }); - var token_buffer: [64]u8 = undefined; - const token = try std.fmt.bufPrint(&token_buffer, "{{\"access_token\":\"tok-{d}\",\"expires_in\":3600}}", .{self.granted.fetchAdd(1, .acq_rel) + 1}); - return req.respond(token, .{ .keep_alive = false }); - } - - if (!std.mem.eql(u8, req.head.target, "/bucket/object?alt=media")) return error.UnexpectedTarget; - _ = self.object_requests.fetchAdd(1, .acq_rel); - var authorization: []const u8 = ""; - var range: ?[]const u8 = null; - var headers = req.iterateHeaders(); - while (headers.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "authorization")) authorization = header.value; - if (std.ascii.eqlIgnoreCase(header.name, "range")) range = header.value; - } - var expected_buffer: [32]u8 = undefined; - const expected = try std.fmt.bufPrint(&expected_buffer, "Bearer tok-{d}", .{self.granted.load(.acquire)}); - if (!std.mem.eql(u8, authorization, expected)) _ = self.stale_requests.fetchAdd(1, .acq_rel); - if (req.head.method == .HEAD) return req.respond(self.object, .{ .keep_alive = false }); - - const spec = (range orelse return error.MissingRange)["bytes=".len..]; - const dash = std.mem.indexOfScalar(u8, spec, '-') orelse return error.InvalidRange; - const first = try std.fmt.parseInt(usize, spec[0..dash], 10); - const last = try std.fmt.parseInt(usize, spec[dash + 1 ..], 10); - var content_range_buffer: [64]u8 = undefined; - return req.respond(self.object[first .. last + 1], .{ - .status = .partial_content, - .keep_alive = false, - .extra_headers = &.{.{ - .name = "Content-Range", - .value = try std.fmt.bufPrint(&content_range_buffer, "bytes {d}-{d}/{d}", .{ first, last, self.object.len }), - }}, - }); - } -}; - -/// A GCS whose tokens and object come from a `TokenServer`. It must not -/// move once set up. -const TokenFixture = struct { - object: [4096]u8 = undefined, - server: TokenServer = undefined, - client: std.http.Client = undefined, - gcs: GCS = undefined, - endpoint_buffer: [32]u8 = undefined, - token_url_buffer: [48]u8 = undefined, - - fn setUp( - self: *TokenFixture, - credentials: @FieldType(GCS.Options, "credentials"), - answers: []const std.http.Status, - max_retries: usize, - ) !void { - const io = std.testing.io; - for (&self.object, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% 3); - self.server = try .init(io, &self.object, answers); - errdefer self.server.deinit(io); - try self.server.group.concurrent(io, TokenServer.run, .{ &self.server, io }); - self.client = .{ .allocator = std.testing.allocator, .io = io }; - errdefer self.client.deinit(); - const port = self.server.tcp.socket.address.getPort(); - const token_uri: std.Uri = try .parse(try std.fmt.bufPrint(&self.token_url_buffer, "http://127.0.0.1:{d}/token", .{port})); - self.gcs = try .init(std.testing.allocator, io, &self.client, .{ - .credentials = credentials, - .endpoint_url = try std.fmt.bufPrint(&self.endpoint_buffer, "http://127.0.0.1:{d}", .{port}), - .retry = .{ - .max_retries = max_retries, - .initial_delay = .fromMilliseconds(5), - .max_delay = .fromMilliseconds(5), - .max_hold = .fromMilliseconds(20), - }, - }); - self.gcs.config.metadata_uri = token_uri; - self.gcs.config.token_uri = token_uri; - } - - fn tearDown(self: *TokenFixture) void { - self.gcs.deinit(); - self.client.deinit(); - self.server.deinit(std.testing.io); - } - - /// Stops the server and returns the first error it met. - fn stop(self: *TokenFixture) !void { - self.server.group.cancel(std.testing.io); - if (self.server.first_error.get()) |err| return err; - } - - /// `readers` concurrent reads of 256 bytes each, started together; - /// returns how many failed. Every read that succeeds must match. - fn readConcurrently(self: *TokenFixture, file: std.Io.File, comptime readers: usize) !usize { - const io = std.testing.io; - const Reader = struct { - fn run(gcs_io: std.Io, file_: std.Io.File, output: []u8, offset: u64, go: *std.Io.Event, result: *anyerror!usize) void { - go.wait(gcs_io) catch |err| { - result.* = err; - return; - }; - result.* = file_.readPositional(gcs_io, &.{output}, offset); - } - }; - var outputs: [readers][256]u8 = undefined; - var results: [readers]anyerror!usize = undefined; - var go: std.Io.Event = .unset; - var group: std.Io.Group = .init; - defer group.cancel(io); - for (&outputs, &results, 0..) |*output, *result, i| { - try group.concurrent(io, Reader.run, .{ self.gcs.io(), file, output, i * output.len, &go, result }); - } - go.set(io); - try group.await(io); - - var failed: usize = 0; - for (&outputs, results, 0..) |*output, result, i| { - const n = result catch { - failed += 1; - continue; - }; - try std.testing.expectEqual(output.len, n); - try std.testing.expectEqualSlices(u8, self.object[i * output.len ..][0..output.len], output); - } - return failed; - } -}; - -test "GCS refreshes an expired token once for concurrent readers, and each sends the new one" { - var fixture: TokenFixture = .{}; - try fixture.setUp(.{ .metadata_server = {} }, &.{.ok}, 0); - defer fixture.tearDown(); - const server = &fixture.server; - const gcs_io = fixture.gcs.io(); - const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); - defer file.close(gcs_io); - try std.testing.expectEqual(1, server.token_requests.load(.acquire)); - - fixture.gcs.token.expires_at = .zero; - try std.testing.expectEqual(0, try fixture.readConcurrently(file, 16)); - try fixture.stop(); - try std.testing.expectEqual(2, server.token_requests.load(.acquire)); - try std.testing.expectEqual(17, server.object_requests.load(.acquire)); - try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); -} - -test "GCS fails the readers queued behind a failed token refresh without repeating it" { - const cases = [_]struct { status: std.http.Status, max_retries: usize, token_requests: usize }{ - // The open's request, then one refused refresh. - .{ .status = .unauthorized, .max_retries = 0, .token_requests = 2 }, - // The open's request, then one refresh and its 2 retries. - .{ .status = .internal_server_error, .max_retries = 2, .token_requests = 4 }, - }; - for (cases) |case| { - var fixture: TokenFixture = .{}; - try fixture.setUp(.{ .metadata_server = {} }, &.{ .ok, case.status }, case.max_retries); - defer fixture.tearDown(); - const server = &fixture.server; - const gcs_io = fixture.gcs.io(); - const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); - defer file.close(gcs_io); - - fixture.gcs.token.expires_at = .zero; - try std.testing.expectEqual(16, try fixture.readConcurrently(file, 16)); - try fixture.stop(); - try std.testing.expectEqual(case.token_requests, server.token_requests.load(.acquire)); - try std.testing.expectEqual(1, server.object_requests.load(.acquire)); - } -} - -test "GCS resends the ADC refresh form on a retry, and a throttled token endpoint holds the backend" { - var json: std.Io.Reader = .fixed( - \\{"type":"authorized_user","client_id":"cid.apps.googleusercontent.com","client_secret":"GOCSPX-sec","refresh_token":"1//0abc-def_ghi","account":"","universe_domain":"googleapis.com"} - ); - var fixture: TokenFixture = .{}; - try fixture.setUp(.{ .json = &json }, &.{ .too_many_requests, .internal_server_error, .ok }, 2); - defer fixture.tearDown(); - const server = &fixture.server; - server.delay = .zero; - server.form = "grant_type=refresh_token&client_id=cid.apps.googleusercontent.com&client_secret=GOCSPX-sec&refresh_token=1//0abc-def_ghi"; - const gcs_io = fixture.gcs.io(); - const file = try std.Io.Dir.openFile(.cwd(), gcs_io, "bucket/object", .{}); - defer file.close(gcs_io); - - try std.testing.expectEqual(0, try fixture.readConcurrently(file, 4)); - try fixture.stop(); - const stats = &fixture.gcs.governor.stats; - try std.testing.expectEqual(1, stats.holds.load(.acquire)); - try std.testing.expectEqual(1, stats.retries.load(.acquire)); - try std.testing.expectEqual(3, server.token_requests.load(.acquire)); - try std.testing.expectEqual(0, server.bad_forms.load(.acquire)); - try std.testing.expectEqual(0, server.stale_requests.load(.acquire)); -} diff --git a/vfs/hf.zig b/vfs/hf.zig index 2989d61f..8770fe18 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -2,13 +2,160 @@ const std = @import("std"); const stdx = @import("stdx"); -const range_read = @import("range_read.zig"); -const request = @import("request.zig"); +const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/hf"); +const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + uri: std.Uri, + url: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + pub const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout => .retry(), + .too_many_requests => b: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const p = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, p, "t=")) { + const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; + break :b .{ .retry_after = .fromSeconds(seconds) }; + } + } + } + } + break :b .retry(); + }, + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; +}; + pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; @@ -53,10 +200,16 @@ const RepoKey = struct { }; pub const HF = struct { - pub const Options = struct { - /// Sent as a bearer token to the Hub itself, never to its CDN. - token: ?[]const u8 = null, - retry: request.Retry = .{}, + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + // Chunk size needs to be big enough to avoid hitting the rate limits of HF. + .chunk_size = 32 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, }; pub const Repo = struct { @@ -90,17 +243,13 @@ pub const HF = struct { type: Type, uri: []const u8, - download_url: ?[]const u8, pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, download_url: ?[]const u8, size: u64) !Handle { - const owned_uri = try allocator.dupe(u8, uri); - errdefer allocator.free(owned_uri); + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { return .{ .type = handle_type, - .uri = owned_uri, - .download_url = if (download_url) |url| try allocator.dupe(u8, url) else null, + .uri = try allocator.dupe(u8, uri), .pos = 0, .size = size, }; @@ -108,37 +257,39 @@ pub const HF = struct { pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { allocator.free(self.uri); - if (self.download_url) |url| allocator.free(url); } }; allocator: std.mem.Allocator, - /// Guards the handle table and the handles' download URLs. mutex: std.Io.Mutex = .init, + client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, - governor: request.Governor, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, - /// Guards `trees`. A tree fetch is a governed request that may wait out - /// a hold or retry for minutes, so it runs under this lock, not under - /// `mutex`, which every read takes. - trees_mutex: std.Io.Mutex = .init, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HF { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + var self: HF = .{ .allocator = allocator, .base = .init(inner), - .authorization = if (options.token) |token| blk: { + .client = http_client, + .authorization = if (hf_token) |token| blk: { break :blk .{ .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), }; } else blk: { break :blk .default; }, - .governor = .init(inner, http_client, "hf", .server_failure, options.retry), + .read_pool = read_pool, }; errdefer switch (self.authorization) { .default, .omit => {}, @@ -170,11 +321,13 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, .{ .token = hf_token }); + return init(allocator, inner, http_client, hf_token, .{}); } pub fn deinit(self: *HF) void { - self.governor.logTotals(); + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -230,16 +383,6 @@ pub const HF = struct { }; } - pub fn backend(self: *HF) Backend { - return .{ - .io = self.io(), - .read_hints = .{ - .read_chunk_size = 32 * 1024 * 1024, - .high_latency = true, - }, - }; - } - fn openHandle(self: *HF) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -286,36 +429,39 @@ pub const HF = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - /// The repository's listing, fetched once. The lock is held across the - /// fetch, so every open, stat or listing of this backend waits for it, - /// cached or not. The map is read and written under `trees_mutex`, since - /// an insert may move its entries; the nodes themselves never move once - /// fetched, so the returned slice stays valid without the lock. - fn getOrFetchTree(self: *HF, repo: Repo) ![]const TreeNode { + fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { const key = repo.toKey(); var cache_key_buffer: [512]u8 = undefined; + const cache_key_lookup = try key.format(&cache_key_buffer); - try self.trees_mutex.lock(self.base.inner); - defer self.trees_mutex.unlock(self.base.inner); + if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; - if (self.trees.get(cache_key_lookup)) |tree| return tree.items; - - // Room first: once fetched, the listing cannot fail to be kept. - try self.trees.ensureUnusedCapacity(self.allocator, 1); const cache_key = try key.allocFormat(self.allocator); errdefer self.allocator.free(cache_key); - const tree = try self.fetchTreeFromAPI(repo); - self.trees.putAssumeCapacityNoClobber(cache_key, tree); - return tree.items; + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + const gop = try self.trees.getOrPut(self.allocator, cache_key); + errdefer _ = self.trees.remove(cache_key); + + if (gop.found_existing) { + self.allocator.free(cache_key); + return gop.value_ptr; + } + + const tree_root = try self.fetchTreeFromAPI(repo); + gop.value_ptr.* = tree_root; + + return gop.value_ptr; } - /// The repository listing, through the governed loop like every other - /// request. It runs under `trees_mutex` from `getOrFetchTree`, never - /// under `mutex`, so reads of open files go on while it waits. fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { var url_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ .repo = repo.repo, .model = repo.model, @@ -325,64 +471,40 @@ pub const HF = struct { const uri = try std.Uri.parse(url); log.info("Fetching from HF: {s}", .{url}); - var fetch: TreeFetch = .{ .hf = self, .uri = uri }; - return request.perform(std.ArrayList(TreeNode), &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &fetch, TreeFetch.attempt); - } + var req = try self.client.request(.GET, uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = self.authorization, + }, + }); + defer req.deinit(); - const TreeFetch = struct { - hf: *HF, - uri: std.Uri, + try req.sendBodiless(); - fn attempt(self: *TreeFetch, current: request.Attempt) anyerror!request.Outcome(std.ArrayList(TreeNode)) { - return request.exchange(std.ArrayList(TreeNode), current, self.uri, .{ - .authorization = self.hf.authorization, - .open = true, - .redirects = .follow, - }, self, TreeFetch.consume); + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to fetch tree: status={}", .{res.head.status}); + return error.RequestFailed; } - fn consume(self: *TreeFetch, res: *std.http.Client.Response) anyerror!std.ArrayList(TreeNode) { - const allocator = self.hf.allocator; - const body = try res.reader(&.{}).readAlloc(allocator, res.head.content_length orelse return error.MissingContentLength); - defer allocator.free(body); + const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); + defer self.allocator.free(body); - const parsed = try std.json.parseFromSlice( - []API.Tree, - allocator, - body, - .{ .ignore_unknown_fields = true }, - ); - defer parsed.deinit(); + const parsed = try std.json.parseFromSlice( + []API.Tree, + self.allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .empty; - errdefer { - for (tree_root.items) |*node| node.deinit(allocator); - tree_root.deinit(allocator); - } - for (parsed.value) |item| { - try insertTreeNode(allocator, &tree_root, item); - } - return tree_root; + var tree_root: std.ArrayList(TreeNode) = .empty; + for (parsed.value) |item| { + try insertTreeNode(self.allocator, &tree_root, item); } - }; - /// The URL the file's bytes are read from, copied into `out_buffer`: - /// where the Hub's redirect chain ends, a signed CDN URL for an LFS file. - fn resolveDownloadUrl(self: *HF, repo: Repo, out_buffer: []u8) ![]const u8 { - var url_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - // Logs name the file: past the first hop the URL is a signed one, - // which is a credential. - var target_buffer: [1024]u8 = undefined; - const target = std.fmt.bufPrint(&target_buffer, "{s}/{s}@{s}/{s}", .{ repo.repo, repo.model, repo.rev, repo.path }) catch repo.path; - var chain: request.HeadChain = .{}; - const end = try chain.resolve(&self.governor, target, url, 8, self, hubAuthorization); - return std.fmt.bufPrint(out_buffer, "{s}", .{end.url}); - } - - /// Credentials only travel to the Hub itself, never to its CDN. - fn hubAuthorization(self: *HF, url: []const u8) std.http.Client.Request.Headers.Value { - return if (std.mem.startsWith(u8, url, "https://huggingface.co/")) self.authorization else .omit; + return tree_root; } fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { @@ -448,16 +570,16 @@ pub const HF = struct { return null; } - fn getSizeFromTree(tree: []const TreeNode, path: []const u8) ?u64 { + fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { if (path.len == 0) { var total: u64 = 0; - for (tree) |*node| { + for (tree.items) |*node| { if (node.kind == .file) total += node.size; } return total; } - const node = findNode(tree, path) orelse return null; + const node = findNode(tree.items, path) orelse return null; return node.size; } @@ -518,9 +640,9 @@ pub const HF = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { - error.Canceled => return error.Canceled, - error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { @@ -543,11 +665,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, full_path, null, size) catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; return .{ .handle = @intCast(idx) }; } @@ -576,16 +698,16 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; - const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); - // The repository itself is a directory. - if (repo.path.len == 0) return std.Io.Dir.StatFileError.IsDir; - const node = findNode(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; + const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; + if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; return .{ .inode = @intCast(0), .nlink = 0, - .size = node.size, + .size = size, .permissions = .fromMode(0o444), .kind = .file, .atime = null, @@ -602,11 +724,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; - const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; if (repo.path.len == 0) return; - const node = findNode(tree, repo.path); + const node = findNode(tree.items, repo.path); if (node == null) return std.Io.Dir.AccessError.FileNotFound; } @@ -617,17 +739,11 @@ pub const HF = struct { const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; - const tree = self.getOrFetchTree(repo) catch |err| return self.governor.openFailure(full_path, err); - // A directory, the repository itself included, has no download URL. - if (repo.path.len == 0) return std.Io.File.OpenError.IsDir; - const node = findNode(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; - if (node.kind == .directory) return std.Io.File.OpenError.IsDir; + const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; - var download_url_buffer: [16 * 1024]u8 = undefined; - // Logged by the file's path: its signed URL would be a credential. - const download_url = self.resolveDownloadUrl(repo, &download_url_buffer) catch |err| return self.governor.openFailure(full_path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, full_path, download_url, node.size) catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } @@ -649,12 +765,9 @@ pub const HF = struct { const handle = self.getDirHandle(reader.dir); const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - const tree = self.getOrFetchTree(repo) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => return std.Io.Dir.Reader.Error.Unexpected, - }; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; - const children = findDirChildren(tree, repo.path) orelse &.{}; + const children = findDirChildren(tree.items, repo.path) orelse &.{}; self.dir_read_states.put(self.allocator, reader, .{ .children = children, @@ -732,18 +845,20 @@ pub const HF = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos +| @abs(relative_offset) + handle.pos + @as(u64, @intCast(relative_offset)) else - handle.pos -| @abs(relative_offset); + handle.pos - @as(u64, @intCast(-relative_offset)); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -762,125 +877,44 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - // Another read of the file may replace the handle's URL - // (`DataRequest.reresolve`): each read sends its own copy. - const url = url: { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - break :url try self.allocator.dupe(u8, handle.download_url orelse return error.InvalidHandle); + var read_size: usize = 0; + for (data) |buf| read_size += buf.len; + read_size = @intCast(@min(handle.size - offset, read_size)); + if (read_size == 0) return 0; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + const repo: Repo = try .parse(handle.uri); + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + const uri: std.Uri = try .parse(url); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .uri = uri, + .url = url, + .authorization = self.authorization, }; - defer self.allocator.free(url); - var download: DataRequest = .{ .hf = self, .handle = handle, .uri = try .parse(url), .url = url }; - defer if (download.re_resolved) |owned| self.allocator.free(owned); - return range_read.performRangeRead(&self.governor, .{ - .request = .{ - // The file, not its signed URL: that is a credential, and - // a re-resolve replaces it. - .target = handle.uri, - .key = request.authorityOf(download.uri), - // A signed CDN URL can expire while the backend is held; - // the CDN then answers 401 or 403, which is not retryable in - // general but is here, once, after re-resolving it. - .retry_once = &.{ .unauthorized, .forbidden }, - }, - }, &download, DataRequest.prepare, data, offset, handle.size); + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; + } + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; } - - const DataRequest = struct { - hf: *HF, - handle: *Handle, - /// Parsed from `url`, whose bytes it slices. - uri: std.Uri, - /// The read's own copy of the URL it sends. - url: []const u8, - /// The URL `reresolve` gave this read, owned by the read; at most - /// one per read. - re_resolved: ?[]const u8 = null, - - fn prepare(self: *DataRequest, attempt: request.Attempt, _: std.http.Header) anyerror!range_read.PreparedRequest { - if (attempt.previous_status) |status| switch (status) { - .unauthorized, .forbidden => { - if (self.re_resolved != null) return error.PermissionDenied; - try self.reresolve(); - }, - else => {}, - }; - return .{ - .uri = self.uri, - .authorization = self.hf.hubAuthorization(self.url), - }; - } - - /// Resolves the download URL again through the governed HEAD chain - /// and stores a copy on the handle, so later reads of the same file - /// use the fresh one too. Reads copy the handle's URL under - /// `hf.mutex`, so none still uses the one this replaces. - fn reresolve(self: *DataRequest) !void { - const allocator = self.hf.allocator; - const repo = try Repo.parse(self.handle.uri); - var url_buffer: [16 * 1024]u8 = undefined; - const resolved = try self.hf.resolveDownloadUrl(repo, &url_buffer); - // Everything that can fail comes before anything is handed over. - const owned = try allocator.dupe(u8, resolved); - errdefer allocator.free(owned); - const uri: std.Uri = try .parse(owned); - const for_handle = try allocator.dupe(u8, resolved); - { - self.hf.mutex.lockUncancelable(self.hf.base.inner); - defer self.hf.mutex.unlock(self.hf.base.inner); - if (self.handle.download_url) |previous| allocator.free(previous); - self.handle.download_url = for_handle; - } - self.re_resolved = owned; - self.url = owned; - self.uri = uri; - log.warn("hf: re-resolved the download URL of {s}", .{self.handle.uri}); - } - }; }; - -test "HF reports a path absent from the repository listing as FileNotFound" { - const allocator = std.testing.allocator; - var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; - defer client.deinit(); - var hf: HF = try .init(allocator, std.testing.io, &client, .{}); - defer hf.deinit(); - - // The listing `getOrFetchTree` would keep, so no request is sent. - var tree: std.ArrayList(TreeNode) = .empty; - try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "config.json", .type = "file", .size = 2 }); - try HF.insertTreeNode(allocator, &tree, .{ .oid = "", .path = "nested/model.safetensors", .type = "file", .size = 8 }); - try hf.trees.put(allocator, try allocator.dupe(u8, "owner/model@main"), tree); - - const io = hf.io(); - try std.testing.expectEqual(@as(u64, 2), (try std.Io.Dir.statFile(.cwd(), io, "owner/model/config.json", .{})).size); - try std.Io.Dir.access(.cwd(), io, "owner/model/nested/model.safetensors", .{}); - try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model/nested", .{})); - try std.testing.expectError(error.IsDir, std.Io.Dir.statFile(.cwd(), io, "owner/model", .{})); - // Opening a directory as a file sends no request. - try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model/nested", .{})); - try std.testing.expectError(error.IsDir, std.Io.Dir.openFile(.cwd(), io, "owner/model", .{})); - - try std.testing.expectError(error.FileNotFound, std.Io.Dir.statFile(.cwd(), io, "owner/model/missing.json", .{})); - try std.testing.expectError(error.FileNotFound, std.Io.Dir.access(.cwd(), io, "owner/model/missing.json", .{})); - try std.testing.expectError(error.FileNotFound, std.Io.Dir.openFile(.cwd(), io, "owner/model/missing.json", .{})); - try std.testing.expectError(error.FileNotFound, std.Io.Dir.openDir(.cwd(), io, "owner/model/missing", .{})); -} - -test "HF sends its token to the Hub only" { - const allocator = std.testing.allocator; - var client: std.http.Client = .{ .allocator = allocator, .io = std.testing.io }; - defer client.deinit(); - var hf: HF = try .init(allocator, std.testing.io, &client, .{ .token = "hf_token" }); - defer hf.deinit(); - - try std.testing.expectEqualStrings("Bearer hf_token", hf.hubAuthorization("https://huggingface.co/owner/model/resolve/main/model.safetensors").override); - // Not the Hub: the CDNs its resolve URLs redirect to, a look-alike - // host, plain http. - for ([_][]const u8{ - "https://cas-bridge.xethub.hf.co/xet-bridge-us/0123?X-Amz-Signature=abc", - "https://cdn-lfs.huggingface.co/repos/00/11/0123", - "https://huggingface.co.evil.com/owner/model/resolve/main/model.safetensors", - "http://huggingface.co/owner/model/resolve/main/model.safetensors", - }) |url| try std.testing.expect(hf.hubAuthorization(url) == .omit); -} diff --git a/vfs/http.zig b/vfs/http.zig index a9765d0f..9054983e 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -3,18 +3,10 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const Backend = @import("base.zig").Backend; -const range_read = @import("range_read.zig"); -const request = @import("request.zig"); const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { - pub const Options = struct { - protocol: Protocol, - retry: request.Retry = .{}, - }; - const Handle = struct { pub const Type = enum { file, @@ -47,23 +39,22 @@ pub const HTTP = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, + client: *std.http.Client, protocol: Protocol, - governor: request.Governor, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !HTTP { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { return .{ .allocator = allocator, .base = .init(inner), - .protocol = options.protocol, - .governor = .init(inner, http_client, "http", .server_failure, options.retry), + .client = http_client, + .protocol = protocol, }; } pub fn deinit(self: *HTTP) void { - self.governor.logTotals(); var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -103,13 +94,6 @@ pub const HTTP = struct { }; } - pub fn backend(self: *HTTP) Backend { - return .{ - .io = self.io(), - .read_hints = .{ .high_latency = true }, - }; - } - fn openHandle(self: *HTTP) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -160,9 +144,9 @@ pub const HTTP = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { - error.Canceled => return error.Canceled, - error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { @@ -209,9 +193,7 @@ pub const HTTP = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; return .{ .inode = @intCast(0), @@ -226,19 +208,15 @@ pub const HTTP = struct { }; } - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; - _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); - } + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -304,18 +282,20 @@ pub const HTTP = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - // Saturates at both ends; `@abs` takes `minInt(i64)` too. handle.pos = if (relative_offset >= 0) - handle.pos +| @abs(relative_offset) + handle.pos + @as(u64, @intCast(relative_offset)) else - handle.pos -| @abs(relative_offset); + handle.pos - @as(u64, @intCast(-relative_offset)); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { @@ -331,31 +311,117 @@ pub const HTTP = struct { return path.len; } - /// The size of the object at `url` (a resolved path without its scheme), - /// from a HEAD of the end of its redirect chain, as far as a read's GET - /// follows it (`request.max_redirects`). - fn fetchSize(self: *HTTP, url: []const u8) !u64 { - var url_buffer: [8 * 1024]u8 = undefined; - const full_url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), url }); - var chain: request.HeadChain = .{}; - const end = try chain.resolve(&self.governor, url, full_url, request.max_redirects, {}, urlAuthorization); - return end.size orelse error.MissingContentLength; - } + fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + var aux_buffer: []u8 = &redirect_buffer; - /// Only what the URL carries: std's basic authorization from its - /// userinfo, as on every GET. - fn urlAuthorization(_: void, _: []const u8) std.http.Client.Request.Headers.Value { - return .default; + const scheme = @tagName(self.protocol); + const url = try self.resolvePath(dir, sub_path, &path_buffer); + const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); + + var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; + while (true) { + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + switch (res.head.status.class()) { + .server_error, .client_error => { + log.err("Failed to fetch tree size for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.ServerError; + }, + .informational => return error.UnexpectedStatus, + .success => return res.head.content_length.?, + .redirect => { + const location = res.head.location.?; + @memcpy(aux_buffer[0..location.len], location); + uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; + continue; + }, + } + } } fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + var url_buffer: [8 * 1024]u8 = undefined; const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); const uri: std.Uri = try .parse(url); - return range_read.performRangeRead(&self.governor, .{ - .request = .{ .target = url, .key = request.authorityOf(uri) }, - // The object `fetchSize` found at the end of the chain. - .redirects = .follow, - }, &uri, range_read.static, data, offset, handle.size); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to perform read for {s}", .{handle.uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; } }; diff --git a/vfs/http_acceptance_test.zig b/vfs/http_acceptance_test.zig deleted file mode 100644 index a09464a5..00000000 --- a/vfs/http_acceptance_test.zig +++ /dev/null @@ -1,704 +0,0 @@ -const std = @import("std"); - -const GCS = @import("gcs.zig").GCS; -const HTTP = @import("http.zig").HTTP; -const MockServer = @import("mock_server.zig").MockServer; -const range_read = @import("range_read.zig"); -const vfs_request = @import("request.zig"); -const S3 = @import("s3.zig").S3; - -/// A retry configuration that never sleeps, for the tests that count -/// attempts rather than time. -fn instantRetries(max_retries: usize) vfs_request.Retry { - return .{ - .max_retries = max_retries, - .initial_delay = .fromNanoseconds(0), - .max_delay = .fromNanoseconds(0), - .max_hold = .fromNanoseconds(0), - .throttle_budget = .fromSeconds(300), - }; -} - -/// A retry configuration whose retries and holds are short enough for a test. -fn fastRetry(initial_ms: i64, budget_ms: i64) vfs_request.Retry { - return .{ - .max_retries = 5, - .initial_delay = .fromMilliseconds(initial_ms), - .max_delay = .fromMilliseconds(initial_ms), - .max_hold = .fromSeconds(30), - .throttle_budget = .fromMilliseconds(budget_ms), - }; -} - -/// A `MockServer` serving `object`, an HTTP backend with a client of its -/// own, and `/object` open through that backend. Initialized in place: the -/// backend points at the client and the server must not move once started. -const Fixture = struct { - server: MockServer, - client: std.http.Client, - http: HTTP, - file: std.Io.File, - - fn init(self: *Fixture, object: []const u8, server_opts: MockServer.Options, retry: vfs_request.Retry) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - self.server = try .init(io, object, server_opts); - errdefer self.server.deinit(io); - try self.server.start(io); - self.client = .{ .allocator = allocator, .io = io }; - errdefer self.client.deinit(); - self.http = try .init(allocator, io, &self.client, .{ .protocol = .http, .retry = retry }); - errdefer self.http.deinit(); - var path_buffer: [128]u8 = undefined; - const path = try self.server.path(&path_buffer); - self.file = try std.Io.Dir.openFile(.cwd(), self.http.io(), path, .{ .mode = .read_only }); - } - - fn deinit(self: *Fixture) void { - self.file.close(self.http.io()); - self.http.deinit(); - self.client.deinit(); - self.server.deinit(std.testing.io); - } -}; - -/// Reads `object` back through the fixture's file with `readers` concurrent -/// positional reads of `read_size` bytes, reader `i` at `i * read_size`, -/// all released at once. Fails on the first reader that failed or read -/// other bytes. -fn expectConcurrentReads(fixture: *Fixture, object: []const u8, comptime readers: usize, comptime read_size: usize) !void { - const Reader = struct { - fn run(file: std.Io.File, file_io: std.Io, output: []u8, offset: u64, result: *anyerror!usize, start: *std.Io.Event) void { - start.wait(file_io) catch |err| { - result.* = err; - return; - }; - result.* = file.readPositional(file_io, &.{output}, offset); - } - }; - const io = std.testing.io; - var outputs: [readers][read_size]u8 = undefined; - var results: [readers]anyerror!usize = undefined; - var released: std.Io.Event = .unset; - var group: std.Io.Group = .init; - defer group.cancel(io); - for (&outputs, &results, 0..) |*output, *result, index| { - try group.concurrent(io, Reader.run, .{ fixture.file, fixture.http.io(), output, index * read_size, result, &released }); - } - released.set(io); - try group.await(io); - - for (&outputs, results, 0..) |*output, result, index| { - try std.testing.expectEqual(read_size, try result); - try std.testing.expectEqualSlices(u8, object[index * read_size ..][0..read_size], output); - } -} - -test "generic HTTP keeps a large scattered positional read to one GET" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const offset = 13; - const request_size = 17 * 1024 * 1024 + 257; - - const object = try allocator.alloc(u8, offset + request_size); - defer allocator.free(object); - @memset(object, 0xa5); - const output = try allocator.alloc(u8, request_size); - defer allocator.free(output); - - var fixture: Fixture = undefined; - try fixture.init(object, .{}, .{}); - defer fixture.deinit(); - const server = &fixture.server; - const stats = &fixture.http.governor.stats; - - // Fresh stats for the read: the open's HEAD is a request too. - stats.* = .{}; - const buffers = [_][]u8{ - output[0 .. 4 * 1024 * 1024], - output[4 * 1024 * 1024 .. 10 * 1024 * 1024], - output[10 * 1024 * 1024 ..], - }; - try std.testing.expectEqual( - request_size, - try fixture.file.readPositional(fixture.http.io(), &buffers, offset), - ); - try std.testing.expectEqualSlices(u8, object[offset..], output); - - try server.stop(io); - - try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); -} - -test "generic HTTP follows a relative redirect chain at open and on every read" { - const io = std.testing.io; - - var object: [4096]u8 = undefined; - for (&object, 0..) |*byte, i| byte.* = @truncate(i); - var output: [1000]u8 = undefined; - - // `/object` redirects twice, and the second location resolves against - // the first hop's path. - var fixture: Fixture = undefined; - try fixture.init(&object, .{ - .redirects = &.{ - .{ .path = "/object", .location = "/hop/next" }, - .{ .path = "/hop/next", .location = "../moved" }, - }, - .objects = &.{.{ .path = "/moved", .bytes = &object }}, - }, .{}); - defer fixture.deinit(); - const server = &fixture.server; - const http_io = fixture.http.io(); - - try std.testing.expectEqual(@as(u64, object.len), (try fixture.file.stat(http_io)).size); - const buffers = [_][]u8{&output}; - try std.testing.expectEqual(output.len, try fixture.file.readPositional(http_io, &buffers, 77)); - try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); - - try server.stop(io); - // Only the object answers with bytes; the hops before it redirect. - try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 1), server.get_requests.load(.acquire)); -} - -test "generic HTTP retries serially and reports aggregate retry counters" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const request_size = 2 * 1024 * 1024; - - const object = try allocator.alloc(u8, request_size); - defer allocator.free(object); - @memset(object, 0x3c); - const output = try allocator.alloc(u8, request_size); - defer allocator.free(output); - - var fixture: Fixture = undefined; - try fixture.init(object, .{ .fail_first_gets = 1 }, .{ - .max_retries = 1, - .initial_delay = .fromNanoseconds(0), - .max_delay = .fromNanoseconds(0), - }); - defer fixture.deinit(); - const server = &fixture.server; - const http_io = fixture.http.io(); - const stats = &fixture.http.governor.stats; - - stats.* = .{}; - const one_buffer = [_][]u8{output}; - try std.testing.expectEqual( - request_size, - try fixture.file.readPositional(http_io, &one_buffer, 0), - ); - try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); - try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); - - stats.* = .{}; - try std.testing.expectEqual( - request_size, - try fixture.file.readPositional(http_io, &one_buffer, 0), - ); - try std.testing.expectEqual(@as(u64, 1), stats.physical_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); - try std.testing.expectEqualSlices(u8, object, output); - - try server.stop(io); - try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); -} - -test "generic HTTP physical concurrency does not exceed caller admission" { - const io = std.testing.io; - const admission = 3; - const read_size = 1024; - - var object: [admission * read_size]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index); - - var fixture: Fixture = undefined; - try fixture.init(&object, .{ .barrier_gets = admission }, .{}); - defer fixture.deinit(); - const server = &fixture.server; - const stats = &fixture.http.governor.stats; - - stats.* = .{}; - try expectConcurrentReads(&fixture, &object, admission, read_size); - - try server.stop(io); - - const peak = server.peak_gets.load(.acquire); - try std.testing.expectEqual(@as(usize, 1), server.head_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, admission), peak); - try std.testing.expect(peak <= admission); - try std.testing.expectEqual( - @as(usize, admission), - server.get_requests.load(.acquire), - ); - try std.testing.expectEqual(@as(u64, admission), stats.physical_requests.load(.acquire)); -} - -test "the shared range loop prepares the request once per attempt" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const request_size = 3 * 1024; - - var object: [request_size]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 7); - var output: [request_size]u8 = undefined; - - var server = try MockServer.init(io, &object, .{ .fail_first_gets = 2 }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - - // An S3-style request: as a SigV4 signature must be, it is prepared - // again by the hook for every attempt. - const Signed = struct { - uri: std.Uri, - prepared: usize = 0, - - fn prepare(self: *@This(), _: vfs_request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { - try std.testing.expectEqualStrings("Range", range.name); - self.prepared += 1; - return .{ .uri = self.uri }; - } - }; - var url_buffer: [160]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); - var signed: Signed = .{ .uri = try .parse(url) }; - const spec: range_read.RangeSpec = .{ - .request = .{ .target = url, .key = vfs_request.authorityOf(signed.uri) }, - }; - const buffers = [_][]u8{ output[0..1024], output[1024..] }; - - // No retry budget: one attempt, one hook call, the failure is counted. - var no_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(0)); - try std.testing.expectError( - error.RetriesExhausted, - range_read.performRangeRead(&no_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), - ); - try std.testing.expectEqual(@as(usize, 1), signed.prepared); - try std.testing.expectEqual(@as(u64, 1), no_retry.stats.server_failures.load(.acquire)); - - signed.prepared = 0; - var one_retry: vfs_request.Governor = .init(io, &client, "test", .throttle, instantRetries(1)); - try std.testing.expectEqual( - object.len, - try range_read.performRangeRead(&one_retry, spec, &signed, Signed.prepare, &buffers, 0, object.len), - ); - try std.testing.expectEqual(@as(usize, 2), signed.prepared); - try std.testing.expectEqualSlices(u8, &object, &output); - - try server.stop(io); - - // One GET per hook call. - try std.testing.expectEqual(@as(usize, 3), server.get_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 1), server.peak_gets.load(.acquire)); - const stats = &one_retry.stats; - try std.testing.expectEqual(@as(u64, 2), stats.physical_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.server_failures.load(.acquire)); - try std.testing.expectEqual(@as(u64, request_size), stats.physical_bytes.load(.acquire)); -} - -test "one throttled GET holds every reader of the backend" { - const io = std.testing.io; - const readers = 8; - const read_size = 512; - - var object: [readers * read_size]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index); - - // Every reader's first GET is throttled, all eight answered together (the - // barrier), with a one second reset. - var fixture: Fixture = undefined; - try fixture.init(&object, .{ - .barrier_gets = readers, - .throttle = .{ .first_gets = readers, .retry_after_s = 1 }, - }, fastRetry(50, 300_000)); - defer fixture.deinit(); - const server = &fixture.server; - const stats = &fixture.http.governor.stats; - - try expectConcurrentReads(&fixture, &object, readers, read_size); - - try server.stop(io); - - // One hold covered every reader, the server-named second was honoured, - // and no reader burned a retry for it. - try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, readers), stats.throttles.load(.acquire)); - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(usize, 2 * readers), server.get_requests.load(.acquire)); - const throttled_at = server.first_throttle_ns.load(.acquire); - const resumed_at = server.first_get_after_throttle_ns.load(.acquire); - try std.testing.expect(throttled_at != 0 and resumed_at != 0); - try std.testing.expect(resumed_at - throttled_at >= std.time.ns_per_s); -} - -test "a rate window is absorbed by the hold instead of the retry budget" { - const io = std.testing.io; - const readers = 12; - const read_size = 256; - - var object: [readers * read_size]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 3); - - // Four GETs per 100 ms; twelve readers need three windows at best. - var fixture: Fixture = undefined; - try fixture.init(&object, .{ - .throttle = .{ .window = .{ .gets = 4, .per_ms = 100 } }, - }, fastRetry(50, 300_000)); - defer fixture.deinit(); - const stats = &fixture.http.governor.stats; - - try expectConcurrentReads(&fixture, &object, readers, read_size); - - try fixture.server.stop(io); - - try std.testing.expect(stats.holds.load(.acquire) > 0); - try std.testing.expect(stats.throttles.load(.acquire) > 0); - // A limited server never exhausts a reader's retries: the hold carries - // the whole episode. - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); - try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); -} - -test "a throttled HEAD at open waits the hold out and succeeds" { - const io = std.testing.io; - - var object: [64]u8 = undefined; - @memset(&object, 0x11); - - var fixture: Fixture = undefined; - try fixture.init(&object, .{ - .throttle = .{ .first_heads = 1, .retry_after_s = 0 }, - }, fastRetry(20, 300_000)); - defer fixture.deinit(); - const server = &fixture.server; - const stats = &fixture.http.governor.stats; - try std.testing.expectEqual(@as(u64, object.len), try fixture.file.length(fixture.http.io())); - - try server.stop(io); - - // The open retried its HEAD after the hold, and the hold is the - // backend's, not the read path's. - try std.testing.expectEqual(@as(usize, 2), server.head_requests.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.throttles.load(.acquire)); - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); -} - -test "a 503 holds an object store and only retries a plain HTTP server" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const read_size = 512; - - var object: [read_size]u8 = undefined; - @memset(&object, 0x7e); - var output: [read_size]u8 = undefined; - - // Each backend's first two GETs are answered 503. - var server = try MockServer.init(io, &object, .{ - .throttle = .{ .first_gets = 2, .status = .service_unavailable }, - .objects = &.{ - .{ .path = "/bucket/object", .bytes = &object }, - .{ .path = "/bucket/object?alt=media", .bytes = &object }, - }, - }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - var endpoint_buffer: [64]u8 = undefined; - const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); - var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(5) }); - defer s3.deinit(); - var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(5) }); - defer gcs.deinit(); - var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(5) }); - defer http.deinit(); - - var path_buffer: [128]u8 = undefined; - const backends = [_]struct { std.Io, *const vfs_request.Governor, []const u8, bool }{ - .{ s3.io(), &s3.governor, "bucket/object", true }, - .{ gcs.io(), &gcs.governor, "bucket/object", true }, - .{ http.io(), &http.governor, try server.path(&path_buffer), false }, - }; - for (backends) |backend| { - const backend_io, const governor, const path, const object_store = backend; - const file = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); - defer file.close(backend_io); - server.get_requests.store(0, .release); - try std.testing.expectEqual(read_size, try file.readPositional(backend_io, &.{&output}, 0)); - - const stats = &governor.stats; - if (object_store) { - // A 503 is rate limiting. One reader, two throttles in a row: - // each arms its own hold once the previous one has passed, and - // neither is charged a retry. - try std.testing.expectEqual(@as(u64, 2), stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), stats.throttles.load(.acquire)); - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); - } else { - // A 503 is a server failure: no hold, one charged retry per - // attempt. - try std.testing.expectEqual(@as(u64, 0), stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), stats.server_failures.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), stats.retries.load(.acquire)); - } - } - - try server.stop(io); -} - -test "a server that never lets up fails the request with RateLimited" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const read_size = 256; - - var object: [read_size]u8 = undefined; - @memset(&object, 0x33); - var output: [read_size]u8 = undefined; - - var server = try MockServer.init(io, &object, .{ - .throttle = .{ .always = true, .retry_after_s = 0 }, - }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - - var url_buffer: [160]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); - const uri: std.Uri = try .parse(url); - const buffers = [_][]u8{&output}; - - var governor: vfs_request.Governor = .init(io, &client, "test", .throttle, .{ - .max_retries = 5, - .initial_delay = .fromMilliseconds(20), - .max_delay = .fromMilliseconds(20), - .max_hold = .fromMilliseconds(50), - .throttle_budget = .fromMilliseconds(300), - }); - const started: std.Io.Timestamp = .now(io, .awake); - // The read never runs: the backend is rate limited past its budget. - // `fileReadPositional` maps this to `Unexpected` with its own log line. - try std.testing.expectError(error.RateLimited, range_read.performRangeRead(&governor, .{ - .request = .{ .target = url }, - }, &uri, range_read.static, &buffers, 0, object.len)); - const elapsed = started.durationTo(.now(io, .awake)).nanoseconds; - try std.testing.expect(elapsed >= 300 * std.time.ns_per_ms); - try std.testing.expect(elapsed < 5 * std.time.ns_per_s); - try std.testing.expect(governor.stats.holds.load(.acquire) > 1); - - try server.stop(io); -} - -test "a reader cancelled while the backend is held returns Canceled" { - const io = std.testing.io; - const read_size = 256; - - var object: [read_size]u8 = undefined; - @memset(&object, 0x21); - var output: [read_size]u8 = undefined; - - var fixture: Fixture = undefined; - try fixture.init(&object, .{ - .throttle = .{ .first_gets = 1, .retry_after_s = 30 }, - }, fastRetry(20, 300_000)); - defer fixture.deinit(); - - var failure: std.atomic.Value(u16) = .init(0); - var readers: std.Io.Group = .init; - try readers.concurrent(io, struct { - fn run(file_: std.Io.File, io_: std.Io, out: []u8, failure_: *std.atomic.Value(u16)) void { - const buffers = [_][]u8{out}; - _ = file_.readPositional(io_, &buffers, 0) catch |err| { - failure_.store(@intFromError(err), .release); - return; - }; - } - }.run, .{ fixture.file, fixture.http.io(), output[0..], &failure }); - - // Let the first GET be throttled and the reader settle into the hold. - try io.sleep(.fromMilliseconds(50), .awake); - readers.cancel(io); - try std.testing.expectEqual(@intFromError(error.Canceled), failure.load(.acquire)); - - try fixture.server.stop(io); -} - -test "generic HTTP cancels a GET that receives nothing and retries it as a timeout" { - const io = std.testing.io; - - var object: [4096]u8 = undefined; - for (&object, 0..) |*byte, i| byte.* = @truncate(i); - var output: [1000]u8 = undefined; - - // The first GET waits unanswered for the second: a peer that receives - // the request and sends nothing. - var fixture: Fixture = undefined; - try fixture.init(&object, .{ .barrier_gets = 2 }, .{ - .initial_delay = .fromNanoseconds(0), - .max_delay = .fromNanoseconds(0), - .stall_timeout = .fromMilliseconds(100), - }); - defer fixture.deinit(); - const server = &fixture.server; - const stats = &fixture.http.governor.stats; - - stats.* = .{}; - const buffers = [_][]u8{&output}; - try std.testing.expectEqual(output.len, try fixture.file.readPositional(fixture.http.io(), &buffers, 77)); - try std.testing.expectEqualSlices(u8, object[77..][0..output.len], &output); - try std.testing.expectEqual(@as(u64, 1), stats.timeouts.load(.acquire)); - try std.testing.expectEqual(@as(u64, 1), stats.retries.load(.acquire)); - - // The first GET answers late, to a connection the client closed: that - // write may fail. - server.stop(io) catch |err| if (err != error.WriteFailed) return err; - try std.testing.expectEqual(@as(usize, 2), server.get_requests.load(.acquire)); -} - -test "http, s3 and gs fail open, stat and access of an absent or denied object alike" { - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Case = struct { name: []const u8, status: std.http.Status, expected: anyerror }; - const cases = [_]Case{ - .{ .name = "missing", .status = .not_found, .expected = error.FileNotFound }, - .{ .name = "gone", .status = .gone, .expected = error.FileNotFound }, - .{ .name = "unauthorized", .status = .unauthorized, .expected = error.PermissionDenied }, - .{ .name = "forbidden", .status = .forbidden, .expected = error.PermissionDenied }, - }; - // The target each backend sends for an object: HTTP its path, S3 - // `//`, GCS the same with `?alt=media`. - const targets = [_]struct { []const u8, []const u8 }{ .{ "/", "" }, .{ "/bucket/", "" }, .{ "/bucket/", "?alt=media" } }; - const refusals = comptime refusals: { - var list: [cases.len * targets.len]MockServer.Refusal = undefined; - for (cases, 0..) |case, i| { - for (targets, 0..) |target, j| { - list[i * targets.len + j] = .{ .path = target[0] ++ case.name ++ target[1], .status = case.status }; - } - } - break :refusals list; - }; - - var object: [32]u8 = undefined; - @memset(&object, 0x6b); - var server = try MockServer.init(io, &object, .{ - .objects = &.{ - .{ .path = "/bucket/object", .bytes = &object }, - .{ .path = "/bucket/object?alt=media", .bytes = &object }, - }, - .refusals = &refusals, - }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - var endpoint_buffer: [64]u8 = undefined; - const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); - var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http }); - defer http.deinit(); - var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); - defer s3.deinit(); - var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint }); - defer gcs.deinit(); - - var authority_buffer: [64]u8 = undefined; - const backends = [_]struct { std.Io, []const u8 }{ - .{ http.io(), try std.fmt.bufPrint(&authority_buffer, "{s}/", .{endpoint["http://".len..]}) }, - .{ s3.io(), "bucket/" }, - .{ gcs.io(), "bucket/" }, - }; - for (backends) |backend| { - const backend_io, const prefix = backend; - var path_buffer: [128]u8 = undefined; - const present = try std.fmt.bufPrint(&path_buffer, "{s}object", .{prefix}); - try std.Io.Dir.access(.cwd(), backend_io, present, .{}); - try std.testing.expectEqual(@as(u64, object.len), (try std.Io.Dir.statFile(.cwd(), backend_io, present, .{})).size); - for (cases) |case| { - const path = try std.fmt.bufPrint(&path_buffer, "{s}{s}", .{ prefix, case.name }); - try std.testing.expectError(case.expected, std.Io.Dir.openFile(.cwd(), backend_io, path, .{})); - try std.testing.expectError(case.expected, std.Io.Dir.statFile(.cwd(), backend_io, path, .{})); - try std.testing.expectError(case.expected, std.Io.Dir.access(.cwd(), backend_io, path, .{})); - } - } - - try server.stop(io); -} - -test "http, s3 and gs fail a refused streaming read with InputOutput and seek back no further than 0" { - const allocator = std.testing.allocator; - const io = std.testing.io; - - var object: [64]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 5 +% 1); - // Each backend's first GET is answered 500, and none has a retry left. - var server = try MockServer.init(io, &object, .{ - .fail_first_gets = 3, - .objects = &.{ - .{ .path = "/bucket/object", .bytes = &object }, - .{ .path = "/bucket/object?alt=media", .bytes = &object }, - }, - }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - var endpoint_buffer: [64]u8 = undefined; - const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); - var http = try HTTP.init(allocator, io, &client, .{ .protocol = .http, .retry = instantRetries(0) }); - defer http.deinit(); - var s3 = try S3.init(allocator, io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1", .retry = instantRetries(0) }); - defer s3.deinit(); - var gcs = try GCS.init(allocator, io, &client, .{ .endpoint_url = endpoint, .retry = instantRetries(0) }); - defer gcs.deinit(); - - var path_buffer: [128]u8 = undefined; - const backend_ios = [_]std.Io{ http.io(), s3.io(), gcs.io() }; - const paths = [_][]const u8{ try server.path(&path_buffer), "bucket/object", "bucket/object" }; - var files: [backend_ios.len]std.Io.File = undefined; - var opened: usize = 0; - defer for (files[0..opened], backend_ios[0..opened]) |file, backend_io| file.close(backend_io); - for (&files, backend_ios, paths) |*file, backend_io, path| { - file.* = try std.Io.Dir.openFile(.cwd(), backend_io, path, .{}); - opened += 1; - } - - var head: [8]u8 = undefined; - const head_slices = [_][]u8{&head}; - for (files, backend_ios) |file, backend_io| { - try std.testing.expectError(error.InputOutput, file.readStreaming(backend_io, &head_slices)); - } - for (files, backend_ios) |file, backend_io| { - // Forward, then back by more than the position: at 0. - try backend_io.vtable.fileSeekBy(backend_io.userdata, file, 5); - try backend_io.vtable.fileSeekBy(backend_io.userdata, file, -9); - try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); - try std.testing.expectEqualSlices(u8, object[0..head.len], &head); - // `minInt(i64)`, whose negation overflows, stops at 0 too. - try backend_io.vtable.fileSeekBy(backend_io.userdata, file, std.math.minInt(i64)); - try std.testing.expectEqual(head.len, try file.readStreaming(backend_io, &head_slices)); - try std.testing.expectEqualSlices(u8, object[0..head.len], &head); - } - - try server.stop(io); - try std.testing.expectEqual(@as(usize, 9), server.get_requests.load(.acquire)); -} diff --git a/vfs/mock_server.zig b/vfs/mock_server.zig deleted file mode 100644 index 9206f65b..00000000 --- a/vfs/mock_server.zig +++ /dev/null @@ -1,326 +0,0 @@ -//! A minimal HTTP origin for the VFS and loader tests: one object per path, -//! range reads, and injectable failures and rate limiting. Exported as -//! `VFS.MockServer` so tests above this layer can serve a fixture over a -//! real socket; nothing outside a test build references it. - -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const MockServer = struct { - /// How the server rate limits. A throttled answer carries the status - /// and, when the test names one, a `Retry-After`. - const Throttle = struct { - /// The first N GETs are throttled. - first_gets: usize = 0, - /// The first N HEADs are throttled. - first_heads: usize = 0, - /// Every GET is throttled, forever. - always: bool = false, - status: std.http.Status = .too_many_requests, - retry_after_s: ?u32 = null, - /// At most `gets` GETs per `per_ms` milliseconds; the rest are - /// throttled. - window: ?struct { gets: usize, per_ms: i64 } = null, - }; - - /// One extra path the server serves beside `/object`. - const Object = struct { path: []const u8, bytes: []const u8 }; - - /// A path answered with `status` and no object, on purpose. - pub const Refusal = struct { path: []const u8, status: std.http.Status }; - - /// A path answered with a 302 to `location`, for HEAD and GET alike. - pub const Redirect = struct { path: []const u8, location: []const u8 }; - - pub const Options = struct { - fail_first_gets: usize = 0, - barrier_gets: usize = 0, - throttle: Throttle = .{}, - objects: []const Object = &.{}, - /// Paths answered with a status of their own (404, 403, ...). - refusals: []const Refusal = &.{}, - redirects: []const Redirect = &.{}, - }; - - tcp: std.Io.net.Server, - /// The accept loop and its connections, from `start` to `stop`. - group: std.Io.Group = .init, - object: []const u8, - fail_first_gets: usize, - barrier_gets: usize, - throttle: Throttle, - objects: []const Object, - refusals: []const Refusal, - redirects: []const Redirect, - get_barrier: std.Io.Event = .unset, - head_requests: std.atomic.Value(usize) = .init(0), - get_requests: std.atomic.Value(usize) = .init(0), - active_gets: std.atomic.Value(usize) = .init(0), - peak_gets: std.atomic.Value(usize) = .init(0), - first_error: stdx.FirstError(anyerror) = .{}, - /// Awake nanoseconds of the first throttled answer and of the first - /// answer served after one; zero until they happen. - first_throttle_ns: std.atomic.Value(i64) = .init(0), - first_get_after_throttle_ns: std.atomic.Value(i64) = .init(0), - /// Guards the rate window. - window_mutex: std.Io.Mutex = .init, - window_started_ns: i64 = 0, - window_gets: usize = 0, - - pub fn init( - io: std.Io, - object: []const u8, - opts: Options, - ) !MockServer { - const address: std.Io.net.IpAddress = .{ .ip4 = .loopback(0) }; - return .{ - .tcp = try address.listen(io, .{ .reuse_address = true }), - .object = object, - .fail_first_gets = opts.fail_first_gets, - .barrier_gets = opts.barrier_gets, - .throttle = opts.throttle, - .objects = opts.objects, - .refusals = opts.refusals, - .redirects = opts.redirects, - }; - } - - /// Serves on a task of its own until `stop`. The server must not move - /// from here on. - pub fn start(self: *MockServer, io: std.Io) !void { - try self.group.concurrent(io, run, .{ self, io }); - } - - /// Stops serving, each connection canceled and joined, and returns the - /// first error the server met. - pub fn stop(self: *MockServer, io: std.Io) !void { - self.group.cancel(io); - if (self.first_error.get()) |err| return err; - } - - /// Stops serving if `stop` did not, and closes the listening socket. - pub fn deinit(self: *MockServer, io: std.Io) void { - self.group.cancel(io); - self.tcp.deinit(io); - } - - /// The path of `/object` for an `http` backend: `127.0.0.1:/object`. - pub fn path(self: *const MockServer, buffer: []u8) ![]const u8 { - return std.fmt.bufPrint(buffer, "127.0.0.1:{d}/object", .{self.port()}); - } - - fn bodyFor(self: *const MockServer, target: []const u8) ?[]const u8 { - if (std.mem.eql(u8, target, "/object")) return self.object; - for (self.objects) |object| { - if (std.mem.eql(u8, object.path, target)) return object.bytes; - } - return null; - } - - fn refusalFor(self: *const MockServer, target: []const u8) ?std.http.Status { - for (self.refusals) |refusal| { - if (std.mem.eql(u8, refusal.path, target)) return refusal.status; - } - return null; - } - - fn redirectFor(self: *const MockServer, target: []const u8) ?[]const u8 { - for (self.redirects) |redirect| { - if (std.mem.eql(u8, redirect.path, target)) return redirect.location; - } - return null; - } - - fn nowNs(io: std.Io) i64 { - return @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); - } - - /// Whether this GET is over the rate window. Counts only the requests - /// it lets through, so a throttled one does not consume the budget. - fn overWindow(self: *MockServer, io: std.Io) bool { - const window = self.throttle.window orelse return false; - self.window_mutex.lockUncancelable(io); - defer self.window_mutex.unlock(io); - const now = nowNs(io); - if (now - self.window_started_ns > window.per_ms * std.time.ns_per_ms) { - self.window_started_ns = now; - self.window_gets = 0; - } - if (self.window_gets >= window.gets) return true; - self.window_gets += 1; - return false; - } - - fn respondThrottled(self: *MockServer, io: std.Io, request: *std.http.Server.Request) !void { - _ = self.first_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); - var retry_after_buffer: [16]u8 = undefined; - try request.respond("slow down", .{ - .status = self.throttle.status, - .keep_alive = false, - .extra_headers = if (self.throttle.retry_after_s) |seconds| &.{.{ - .name = "Retry-After", - .value = try std.fmt.bufPrint(&retry_after_buffer, "{d}", .{seconds}), - }} else &.{}, - }); - } - - pub fn port(self: *const MockServer) u16 { - return self.tcp.socket.address.getPort(); - } - - fn run(self: *MockServer, io: std.Io) std.Io.Cancelable!void { - var connections: std.Io.Group = .init; - defer connections.cancel(io); - - while (true) { - const stream = self.tcp.accept(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => { - _ = self.first_error.set(err); - return; - }, - }; - connections.concurrent(io, onConnection, .{ self, stream, io }) catch |err| { - stream.close(io); - _ = self.first_error.set(err); - return; - }; - } - } - - fn onConnection( - self: *MockServer, - stream: std.Io.net.Stream, - io: std.Io, - ) std.Io.Cancelable!void { - defer stream.close(io); - self.handleConnection(stream, io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => _ = self.first_error.set(err), - }; - } - - fn handleConnection(self: *MockServer, stream: std.Io.net.Stream, io: std.Io) !void { - var read_buffer: [8 * 1024]u8 = undefined; - var reader = stream.reader(io, &read_buffer); - var write_buffer: [8 * 1024]u8 = undefined; - var writer = stream.writer(io, &write_buffer); - var server: std.http.Server = .init(&reader.interface, &writer.interface); - var request = try server.receiveHead(); - - if (self.redirectFor(request.head.target)) |location| { - try request.respond("", .{ - .status = .found, - .keep_alive = false, - .extra_headers = &.{.{ .name = "Location", .value = location }}, - }); - return; - } - - const body = self.bodyFor(request.head.target) orelse { - const refusal = self.refusalFor(request.head.target); - try request.respond("refused", .{ - .status = refusal orelse .not_found, - .keep_alive = false, - }); - if (refusal != null) return; - return error.UnexpectedTarget; - }; - - switch (request.head.method) { - .HEAD => { - const head_ordinal = self.head_requests.fetchAdd(1, .acq_rel) + 1; - if (head_ordinal <= self.throttle.first_heads) { - try self.respondThrottled(io, &request); - return; - } - try request.respond(body, .{ .keep_alive = false }); - }, - .GET => { - const range = try parseRequestRange(&request, body.len); - const get_ordinal = self.get_requests.fetchAdd(1, .acq_rel) + 1; - const active = self.active_gets.fetchAdd(1, .acq_rel) + 1; - _ = self.peak_gets.fetchMax(active, .acq_rel); - defer _ = self.active_gets.fetchSub(1, .release); - - if (self.barrier_gets != 0 and get_ordinal <= self.barrier_gets) { - if (get_ordinal == self.barrier_gets) self.get_barrier.set(io); - try self.get_barrier.wait(io); - } - - if (get_ordinal <= self.fail_first_gets) { - try request.respond("retry", .{ - .status = .internal_server_error, - .keep_alive = false, - }); - return; - } - - if (self.throttle.always or get_ordinal <= self.throttle.first_gets or self.overWindow(io)) { - try self.respondThrottled(io, &request); - return; - } - if (self.first_throttle_ns.load(.acquire) != 0) { - _ = self.first_get_after_throttle_ns.cmpxchgStrong(0, nowNs(io), .release, .monotonic); - } - - var content_range_buffer: [96]u8 = undefined; - const content_range = try std.fmt.bufPrint( - &content_range_buffer, - "bytes {d}-{d}/{d}", - .{ range.start, range.end, body.len }, - ); - try request.respond(body[range.start .. range.end + 1], .{ - .status = .partial_content, - .keep_alive = false, - .extra_headers = &.{ - .{ .name = "Content-Range", .value = content_range }, - }, - }); - }, - else => { - try request.respond("method not allowed", .{ - .status = .method_not_allowed, - .keep_alive = false, - }); - return error.UnexpectedMethod; - }, - } - } - - const Range = struct { - start: usize, - end: usize, - }; - - fn parseRequestRange(request: *const std.http.Server.Request, object_len: usize) !Range { - var value: ?[]const u8 = null; - var headers = request.iterateHeaders(); - while (headers.next()) |header| { - if (!std.ascii.eqlIgnoreCase(header.name, "Range")) continue; - if (value != null) return error.DuplicateRange; - value = header.value; - } - - const range_value = value orelse return error.MissingRange; - const prefix = "bytes="; - if (range_value.len <= prefix.len or - !std.ascii.eqlIgnoreCase(range_value[0..prefix.len], prefix)) - { - return error.InvalidRange; - } - const dash = std.mem.indexOfScalar(u8, range_value[prefix.len..], '-') orelse - return error.InvalidRange; - const absolute_dash = prefix.len + dash; - if (absolute_dash == prefix.len or absolute_dash + 1 == range_value.len) - return error.InvalidRange; - - const result: Range = .{ - .start = try std.fmt.parseInt(usize, range_value[prefix.len..absolute_dash], 10), - .end = try std.fmt.parseInt(usize, range_value[absolute_dash + 1 ..], 10), - }; - if (result.end < result.start or result.end >= object_len) return error.InvalidRange; - return result; - } -}; diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig new file mode 100644 index 00000000..70c9d89d --- /dev/null +++ b/vfs/parallel_read.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const InitOpts = struct { + chunk_size: usize, + num_workers: usize, + queue_capacity: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, +}; + +pub const Status = union(enum) { + success, + retry_after: ?std.Io.Duration, + + pub fn retry() Status { + return .{ .retry_after = null }; + } +}; + +pub const BatchState = struct { + pending: std.atomic.Value(u32), + err: std.atomic.Value(u16) = .init(0), + + pub fn failed(batch: *BatchState) bool { + return batch.err.load(.acquire) != 0; + } + + pub fn fail(batch: *BatchState, err: anyerror) void { + _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); + } + + pub fn anyError(batch: *BatchState) ?anyerror { + const err = batch.err.load(.acquire); + return if (err == 0) null else @errorFromInt(err); + } + + pub fn completeOne(batch: *BatchState, io: std.Io) void { + const previous = batch.pending.fetchSub(1, .release); + std.debug.assert(previous > 0); + if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); + } + + pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { + while (true) { + const pending = batch.pending.load(.acquire); + if (pending == 0) return; + io.futexWaitUncancelable(u32, &batch.pending.raw, pending); + } + } +}; + +pub fn Pool(comptime Job: type) type { + return struct { + const Self = @This(); + + group: std.Io.Group = .init, + job_queue: std.Io.Queue(Job), + queue_buf: []Job, + client: *std.http.Client, + prng: std.Random.DefaultPrng, + chunk_size: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, + + pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { + stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); + stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); + stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); + + const queue_buf = try allocator.alloc(Job, opts.queue_capacity); + errdefer allocator.free(queue_buf); + + self.* = .{ + .client = client, + .chunk_size = opts.chunk_size, + .max_retries = opts.max_retries, + .retry_initial_delay = opts.retry_initial_delay, + .retry_max_delay = opts.retry_max_delay, + .prng = .init(blk: { + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + break :blk seed; + }), + .job_queue = .init(queue_buf), + .queue_buf = queue_buf, + }; + errdefer self.group.cancel(io); + + for (0..opts.num_workers) |_| { + try self.group.concurrent(io, worker, .{ self, io }); + } + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { + self.job_queue.close(io); + self.group.cancel(io); + + allocator.free(self.queue_buf); + } + + fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { + while (true) { + const job = pool.job_queue.getOne(io) catch break; + + var attempt: usize = 0; + while (true) { + if (job.batch.state.failed()) break; + const status = job.perform(pool.client) catch |err| { + job.batch.state.fail(err); + break; + }; + + switch (status) { + .success => break, + .retry_after => |duration| { + if (attempt >= pool.max_retries) { + job.batch.state.fail(error.RetriesExhausted); + break; + } + + io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { + job.batch.state.fail(error.RetriesExhausted); + break; + }; + + attempt += 1; + }, + } + continue; + } + + job.batch.state.completeOne(io); + } + } + + fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { + // FullJitter algorithm + const max_delay_ns: i96 = @min( + pool.retry_max_delay.toNanoseconds(), + pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); + return .fromNanoseconds(delay_ns); + } + }; +} + +pub const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +pub fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; +} + +pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested += buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +pub fn readChunk( + reader: *std.Io.Reader, + content_range: ?ContentRange, + file_offset: u64, + data: []const []u8, + chunk_offset: usize, + chunk_len: usize, +) !void { + if (content_range) |cr| { + if (cr.start > file_offset) return error.InvalidContentRange; + if (cr.start < file_offset) { + try reader.discardAll(file_offset - cr.start); + } + } + + var remaining = chunk_len; + var skip = chunk_offset; + + for (data) |buf| { + if (skip >= buf.len) { + skip -= buf.len; + continue; + } + + const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; + try reader.readSliceAll(destination); + remaining -= destination.len; + if (remaining == 0) break; + skip = 0; + } +} diff --git a/vfs/range_read.zig b/vfs/range_read.zig deleted file mode 100644 index 920720e6..00000000 --- a/vfs/range_read.zig +++ /dev/null @@ -1,263 +0,0 @@ -//! The Range specifics of a governed request: the `Range` header, the -//! `Content-Range` validation and the scatter into the caller's buffers. -//! Everything else (the hold, the retries, the classification) is in -//! `request.zig`. - -const std = @import("std"); - -const request = @import("request.zig"); - -const ContentRange = struct { - start: u64, - end: u64, - total: u64, -}; - -/// The backend's part of one GET: the URI, the authorization value and up to -/// `max_extra_headers` headers. The loop appends `Range`. Every slice must -/// stay valid until the attempt completes, so hooks write into storage that -/// outlives the call (the backend's per-read context). -pub const PreparedRequest = struct { - uri: std.Uri, - authorization: std.http.Client.Request.Headers.Value = .default, - extra_headers: []const std.http.Header = &.{}, -}; - -pub const max_extra_headers = 4; - -/// One range read: what identifies it, and whether it follows a redirect. -pub const RangeSpec = struct { - request: request.RequestSpec, - /// The object stores and a resolved download URL never redirect a - /// read; a plain HTTP server may. - redirects: request.Redirects = .forbid, -}; - -/// The `prepare` of a request that is the same on every attempt: a plain -/// GET of `uri`. -pub fn static(uri: *const std.Uri, _: request.Attempt, _: std.http.Header) anyerror!PreparedRequest { - return .{ .uri = uri.* }; -} - -/// One `GET` with `Range: bytes=offset-(offset+size-1)` per attempt through -/// the governed loop, where `size` is what `data` holds of the `file_size` -/// bytes past `offset`; nothing is sent when it is 0. `prepare` is called -/// once per attempt: S3's SigV4 signature covers the `Range` header and a -/// timestamp, a bearer token may have expired during a hold, and a signed -/// download URL may need re-resolving after a 403. -/// The caller holds its read slot through every retry, hold and sleep; -/// backend-local parallel readers would multiply the caller's width and -/// defeat its memory bounds. -pub fn performRangeRead( - governor: *request.Governor, - spec: RangeSpec, - context: anytype, - comptime prepare: fn (@TypeOf(context), request.Attempt, std.http.Header) anyerror!PreparedRequest, - data: []const []u8, - offset: u64, - file_size: u64, -) !usize { - const size = readSize(file_size, offset, data); - if (size == 0) return 0; - - const RangeRead = struct { - context: @TypeOf(context), - redirects: request.Redirects, - range: std.http.Header, - data: []const []u8, - offset: u64, - size: usize, - progress: *std.atomic.Value(u64) = undefined, - - fn attempt(self: *@This(), current: request.Attempt) anyerror!request.Outcome(void) { - self.progress = current.progress; - const prepared = try prepare(self.context, current, self.range); - std.debug.assert(prepared.extra_headers.len <= max_extra_headers); - var headers: [max_extra_headers + 1]std.http.Header = undefined; - @memcpy(headers[0..prepared.extra_headers.len], prepared.extra_headers); - headers[prepared.extra_headers.len] = self.range; - return request.exchange(void, current, prepared.uri, .{ - .authorization = prepared.authorization, - .extra_headers = headers[0 .. prepared.extra_headers.len + 1], - .redirects = self.redirects, - }, self, consume); - } - - fn consume(self: *@This(), res: *std.http.Client.Response) anyerror!void { - // The head bytes are released when the body reader is taken. - const content_range = contentRange(res.head); - return readResponse(res.reader(&.{}), res.head.status, content_range, self.offset, self.data, self.size, self.progress); - } - }; - - var range_buffer: [64]u8 = undefined; - var state: RangeRead = .{ - .context = context, - .redirects = spec.redirects, - .range = .{ - .name = "Range", - .value = std.fmt.bufPrint( - &range_buffer, - "bytes={d}-{d}", - .{ offset, offset + @as(u64, @intCast(size - 1)) }, - ) catch unreachable, - }, - .data = data, - .offset = offset, - .size = size, - }; - try request.perform(void, governor, spec.request, &state, RangeRead.attempt); - governor.stats.recordSuccess(size); - return size; -} - -fn contentRange(head: std.http.Client.Response.Head) ?ContentRange { - var it = head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) return parseContentRange(header.value); - } - return null; -} - -fn parseContentRange(value: []const u8) ?ContentRange { - const prefix = "bytes "; - if (value.len < prefix.len or !std.ascii.eqlIgnoreCase(value[0..prefix.len], prefix)) return null; - const range_and_total = value[prefix.len..]; - const dash = std.mem.indexOfScalar(u8, range_and_total, '-') orelse return null; - const slash = dash + 1 + (std.mem.indexOfScalar(u8, range_and_total[dash + 1 ..], '/') orelse return null); - if (dash == 0 or slash == dash + 1 or slash + 1 == range_and_total.len) return null; - if (std.mem.indexOfScalar(u8, range_and_total[slash + 1 ..], '/') != null) return null; - - const result: ContentRange = .{ - .start = std.fmt.parseInt(u64, range_and_total[0..dash], 10) catch return null, - .end = std.fmt.parseInt(u64, range_and_total[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, range_and_total[slash + 1 ..], 10) catch return null, - }; - if (result.end < result.start or result.end >= result.total) return null; - return result; -} - -fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { - if (offset >= file_size) return 0; - - var requested: usize = 0; - for (data) |buf| requested +|= buf.len; - return @intCast(@min(file_size - offset, requested)); -} - -/// Validates the response against the requested range, discards the prefix -/// of a `200` that ignored `Range`, then scatters `read_size` bytes, counting -/// what it receives in `progress`. -fn readResponse( - reader: *std.Io.Reader, - status: std.http.Status, - content_range: ?ContentRange, - offset: u64, - data: []const []u8, - read_size: usize, - progress: ?*std.atomic.Value(u64), -) !void { - std.debug.assert(read_size != 0); - - const response_start = switch (status) { - .partial_content => blk: { - const cr = content_range orelse return error.InvalidContentRange; - const requested_end = std.math.add(u64, offset, read_size - 1) catch return error.InvalidContentRange; - if (cr.start > offset or cr.end < requested_end) return error.InvalidContentRange; - break :blk cr.start; - }, - .ok => 0, - else => return error.UnexpectedStatus, - }; - - var skip = offset - response_start; - while (skip != 0) { - const len = @min(skip, progress_slice); - try reader.discardAll(len); - count(progress, len); - skip -= len; - } - try readScatter(reader, data, read_size, progress); -} - -/// Bytes read between two progress counts. -const progress_slice = 1 << 20; - -fn count(progress: ?*std.atomic.Value(u64), len: u64) void { - if (progress) |counter| _ = counter.fetchAdd(len, .monotonic); -} - -fn readScatter(reader: *std.Io.Reader, data: []const []u8, len: usize, progress: ?*std.atomic.Value(u64)) !void { - var remaining = len; - for (data) |buf| { - var destination = buf[0..@min(remaining, buf.len)]; - remaining -= destination.len; - while (destination.len != 0) { - const slice = destination[0..@min(destination.len, progress_slice)]; - try reader.readSliceAll(slice); - count(progress, slice.len); - destination = destination[slice.len..]; - } - } - if (remaining != 0) return error.UnexpectedEndOfOutput; -} - -test "Content-Range parsing is strict" { - try std.testing.expectEqual( - ContentRange{ .start = 2, .end = 9, .total = 10 }, - parseContentRange("bytes 2-9/10").?, - ); - try std.testing.expectEqual( - ContentRange{ .start = 2, .end = 9, .total = 10 }, - parseContentRange("ByTeS 2-9/10").?, - ); - try std.testing.expect(parseContentRange("items 2-9/10") == null); - try std.testing.expect(parseContentRange("bytes 2-9/*") == null); - try std.testing.expect(parseContentRange("bytes 9-2/10") == null); - try std.testing.expect(parseContentRange("bytes 2-10/10") == null); - try std.testing.expect(parseContentRange("bytes 2-9/10 trailing") == null); -} - -test "range responses fill scatter buffers" { - var reader: std.Io.Reader = .fixed("23456789"); - var first: [2]u8 = undefined; - var second: [3]u8 = undefined; - try readResponse( - &reader, - .partial_content, - .{ .start = 2, .end = 9, .total = 10 }, - 3, - &.{ &first, &second }, - 5, - null, - ); - try std.testing.expectEqualStrings("34", &first); - try std.testing.expectEqualStrings("567", &second); -} - -test "200 responses that ignore Range are positioned and scattered" { - var reader: std.Io.Reader = .fixed("0123456789"); - var first: [1]u8 = undefined; - var second: [4]u8 = undefined; - try readResponse(&reader, .ok, null, 3, &.{ &first, &second }, 5, null); - try std.testing.expectEqualStrings("3", &first); - try std.testing.expectEqualStrings("4567", &second); -} - -test "partial responses require a covering Content-Range" { - var reader: std.Io.Reader = .fixed("3456"); - var output: [4]u8 = undefined; - try std.testing.expectError(error.InvalidContentRange, readResponse(&reader, .partial_content, null, 3, &.{&output}, 4, null)); - - reader = .fixed("4567"); - try std.testing.expectError( - error.InvalidContentRange, - readResponse(&reader, .partial_content, .{ .start = 4, .end = 7, .total = 10 }, 3, &.{&output}, 4, null), - ); - - reader = .fixed("345"); - try std.testing.expectError( - error.InvalidContentRange, - readResponse(&reader, .partial_content, .{ .start = 3, .end = 5, .total = 10 }, 3, &.{&output}, 4, null), - ); -} diff --git a/vfs/request.zig b/vfs/request.zig deleted file mode 100644 index 9365d460..00000000 --- a/vfs/request.zig +++ /dev/null @@ -1,1212 +0,0 @@ -//! One governed HTTP request loop for every backend: wait out the backend's -//! hold, send, classify. A throttle (429 everywhere, 503 on the object -//! stores) arms a backend-wide hold instead of charging the request's retry -//! budget, so every reader of that backend backs off together and the traffic -//! actually falls; every other retryable failure keeps the per-request -//! backoff. The hold is a deadline, not a permit: nothing is narrowed -//! permanently, recovery is the deadline passing, and a reader's width stays -//! the ceiling it chose, because this loop runs inside its reads. - -const std = @import("std"); - -const log = @import("base.zig").scopedLog(.@"zml/vfs/request"); - -/// The retry knobs of one remote backend: its `Options.retry`. -pub const Retry = struct { - /// Retries per request for failures that are not rate limiting. - max_retries: usize = 5, - initial_delay: std.Io.Duration = .fromMilliseconds(500), - /// Longest backoff before a request's next attempt after a failure that - /// is not a throttle, a server-named delay included. A throttle waits - /// out the backend's hold instead (`max_hold`). - max_delay: std.Io.Duration = .fromSeconds(30), - /// Longest hold one throttle may arm over every request of the backend, - /// a server-named delay included. - max_hold: std.Io.Duration = .fromSeconds(120), - /// A throttling episode with no clean window longer than this fails the - /// backend's requests. - throttle_budget: std.Io.Duration = .fromSeconds(300), - /// An exchange that receives nothing for this long is canceled and - /// retried as a timeout: a peer that stops sending without closing the - /// connection would otherwise hold its reader, and the load waiting for - /// it, forever. Body readers count what they receive - /// (`Attempt.progress`); an exchange that counts nothing (a HEAD, a token - /// exchange) must finish within it. - stall_timeout: std.Io.Duration = .fromSeconds(60), -}; - -/// What a hold covers. One governor holds one scope, its backend instance; -/// the authority is carried so that a per-authority map is a local change in -/// `Governor.holdFor`. -pub const Key = []const u8; - -/// The rate-limit scope of a request: its URI authority. Only the host is -/// taken, since `Governor.holdFor` ignores the key; a per-authority scope -/// would want the port beside it. -pub fn authorityOf(uri: std.Uri) Key { - const host = uri.host orelse return ""; - return switch (host) { - .raw => |raw| raw, - .percent_encoded => |encoded| encoded, - }; -} - -/// One scope's rate-limit state, owned by the governor's mutex. -const Hold = struct { - /// When requests may go again; null when nothing is held. - until: ?std.Io.Timestamp = null, - /// The length of the current hold: a clean window that long ends the - /// episode. - length: std.Io.Duration = .fromNanoseconds(0), - /// Holds armed since the episode began; the backoff base. - consecutive: usize = 0, - episode_started: ?std.Io.Timestamp = null, - /// The episode outlived `throttle_budget`; requests fail until a clean - /// window clears it. - exhausted: bool = false, -}; - -/// One per backend instance: what each of its requests needs, its -/// rate-limit state and its read statistics. -pub const Governor = struct { - pub const Stats = struct { - physical_requests: std.atomic.Value(u64) = .init(0), - physical_bytes: std.atomic.Value(u64) = .init(0), - retries: std.atomic.Value(u64) = .init(0), - transient_retries: std.atomic.Value(u64) = .init(0), - timeouts: std.atomic.Value(u64) = .init(0), - server_failures: std.atomic.Value(u64) = .init(0), - throttles: std.atomic.Value(u64) = .init(0), - retry_delay_ns: std.atomic.Value(u64) = .init(0), - /// Backend-wide holds armed by rate limiting, and the time requests - /// spent waiting them out. - holds: std.atomic.Value(u64) = .init(0), - hold_wait_ns: std.atomic.Value(u64) = .init(0), - - fn recordAttempt(self: *Stats) void { - _ = self.physical_requests.fetchAdd(1, .monotonic); - } - - pub fn recordSuccess(self: *Stats, request_size: usize) void { - _ = self.physical_bytes.fetchAdd(@intCast(request_size), .monotonic); - } - - fn recordFailure(self: *Stats, failure: ReadFailure) void { - switch (failure) { - .transient => _ = self.transient_retries.fetchAdd(1, .monotonic), - .timeout => _ = self.timeouts.fetchAdd(1, .monotonic), - .server_failure => _ = self.server_failures.fetchAdd(1, .monotonic), - .throttle => _ = self.throttles.fetchAdd(1, .monotonic), - } - } - - fn recordRetry(self: *Stats) void { - _ = self.retries.fetchAdd(1, .monotonic); - } - - fn recordRetryDelay(self: *Stats, delay: std.Io.Duration) void { - const delay_ns: u64 = @intCast(@max(delay.nanoseconds, 0)); - _ = self.retry_delay_ns.fetchAdd(delay_ns, .monotonic); - } - - /// One rate-limit hold armed over every request of this backend. - fn recordHold(self: *Stats) void { - _ = self.holds.fetchAdd(1, .monotonic); - } - - /// Time one request spent waiting a hold out. - fn recordHoldWait(self: *Stats, waited: std.Io.Duration) void { - _ = self.hold_wait_ns.fetchAdd(@intCast(@max(waited.nanoseconds, 0)), .monotonic); - } - }; - - io: std.Io, - client: *std.http.Client, - /// The backend's name, for log lines. - name: []const u8, - /// What a 503 means for this backend: rate limiting on the object stores - /// (AWS `SlowDown`, GCS), a server failure elsewhere. - unavailable: ReadFailure, - retry: Retry, - stats: Stats = .{}, - mutex: std.Io.Mutex = .init, - hold: Hold = .{}, - - pub fn init(io: std.Io, client: *std.http.Client, name: []const u8, unavailable: ReadFailure, retry: Retry) Governor { - std.debug.assert(retry.initial_delay.nanoseconds >= 0); - std.debug.assert(retry.max_delay.nanoseconds >= retry.initial_delay.nanoseconds); - std.debug.assert(retry.max_hold.nanoseconds >= retry.initial_delay.nanoseconds); - return .{ .io = io, .client = client, .name = name, .unavailable = unavailable, .retry = retry }; - } - - /// Logs the read totals at debug, once, from the backend's `deinit`; - /// nothing when the backend sent no request. Holds, retries and failures - /// are logged at warn as they happen. - pub fn logTotals(self: *const Governor) void { - const stats = &self.stats; - const requests = stats.physical_requests.load(.acquire); - if (requests == 0) return; - // Under the VFS's scope, `zml/vfs`. - std.log.scoped(.@"zml/vfs").debug("{s} read totals: requests={d}, bytes={Bi:.2}, retries={d}, transient_retries={d}, timeouts={d}, server_failures={d}, throttles={d}, holds={d}, hold_wait_ms={d:.3}, retry_delay_ms={d:.3}", .{ - self.name, - requests, - stats.physical_bytes.load(.acquire), - stats.retries.load(.acquire), - stats.transient_retries.load(.acquire), - stats.timeouts.load(.acquire), - stats.server_failures.load(.acquire), - stats.throttles.load(.acquire), - stats.holds.load(.acquire), - @as(f64, @floatFromInt(stats.hold_wait_ns.load(.acquire))) / std.time.ns_per_ms, - @as(f64, @floatFromInt(stats.retry_delay_ns.load(.acquire))) / std.time.ns_per_ms, - }); - } - - /// A failed open, stat or access of `path` as `OpenError` reports it. - pub fn openFailure(self: *const Governor, path: []const u8, err: anyerror) OpenError { - return switch (err) { - error.FileNotFound => error.FileNotFound, - error.PermissionDenied => error.PermissionDenied, - error.Canceled => error.Canceled, - // Logged by `exchange`. - error.RequestFailed => error.Unexpected, - error.OutOfMemory => error.SystemResources, - else => { - log.err("{s}: looking up {s} failed: {t}", .{ self.name, path, err }); - return error.Unexpected; - }, - }; - } - - /// A failed read of `target` at `offset` as a backend's read reports it: - /// a canceled task stays `Canceled`, since every wait in the governed - /// loop is a cancellation point; any other failure is logged once at - /// err, by `exchange` or here, and is `Unexpected`. - pub fn readFailure(self: *const Governor, target: []const u8, offset: u64, err: anyerror) error{ Canceled, Unexpected } { - return switch (err) { - error.Canceled => error.Canceled, - // Logged by `exchange`. - error.RequestFailed => error.Unexpected, - else => { - log.err("{s}: reading {s} at {d} failed: {t}", .{ self.name, target, offset, err }); - return error.Unexpected; - }, - }; - } - - /// The scope `key` belongs to. One generic `HTTP` instance serves every - /// host of its scheme and one `S3` spans buckets, so per-instance - /// over-reaches there; a per-authority scope replaces this body with a - /// map without touching the loop. - fn holdFor(self: *Governor, key: Key) *Hold { - _ = key; - return &self.hold; - } -}; - -/// One attempt as a backend's request hook sees it. -pub const Attempt = struct { - /// The backend and the request as `perform` was given them, which - /// `exchange` sends and logs with. - governor: *Governor, - spec: RequestSpec, - /// The status the previous attempt returned, for a hook that reacts to - /// one (an expired signed URL answering 403 after a long hold). - previous_status: ?std.http.Status = null, - /// Bytes received so far, for a hook whose body is long: `exchange` - /// cancels an exchange this counter does not move for - /// `Retry.stall_timeout`. - progress: *std.atomic.Value(u64), -}; - -/// How often a watched exchange's progress is checked, per stall timeout. -const stall_checks = 12; - -/// Identity of one request, for the logs and for the classification. -pub const RequestSpec = struct { - /// Object identity for log lines. - target: []const u8, - /// The rate-limit scope: the request URI's authority. - key: Key = "", - /// Statuses this request retries although they are not retryable in - /// general, because its hook can do something about them: a signed - /// download URL that expired during a hold answers 401 or 403, and the - /// hook re-resolves it. Charged to the retry budget. - retry_once: []const std.http.Status = &.{}, -}; - -pub const ReadFailure = enum { - transient, - timeout, - server_failure, - throttle, -}; - -pub const Failure = struct { - failure: ReadFailure, - /// The delay the server named, if any. - delay: ?std.Io.Duration = null, - status: ?std.http.Status = null, -}; - -/// What one attempt produced: a value, or a reason to try again. -pub fn Outcome(comptime T: type) type { - return union(enum) { - done: T, - retry: Failure, - }; -} - -/// The one request loop. `attemptFn` performs a single HTTP exchange (see -/// `exchange`); this drives the hold, the classification and the two budgets -/// around it. A throttle re-runs the attempt without charging a retry, after -/// the hold it armed; any other retryable failure, a stall -/// (`Retry.stall_timeout`) included, sleeps the per-request backoff and charges -/// one of `max_retries`. A cancel of the caller propagates. -pub fn perform( - comptime T: type, - governor: *Governor, - spec: RequestSpec, - context: anytype, - comptime attemptFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), -) anyerror!T { - var charged: usize = 0; - var previous_status: ?std.http.Status = null; - while (true) { - try admit(governor, spec); - governor.stats.recordAttempt(); - var progress: std.atomic.Value(u64) = .init(0); - switch (try attemptFn(context, .{ .governor = governor, .spec = spec, .previous_status = previous_status, .progress = &progress })) { - .done => |value| return value, - .retry => |failure| { - governor.stats.recordFailure(failure.failure); - previous_status = failure.status; - if (failure.failure == .throttle) { - // Not charged to the retry budget: the hold is what - // reduces the traffic, and the episode budget bounds it. - try reportThrottle(governor, spec, failure.delay); - continue; - } - if (charged >= governor.retry.max_retries) return error.RetriesExhausted; - charged += 1; - governor.stats.recordRetry(); - const retry = governor.retry; - const delay = if (failure.delay) |named| - clampDelay(named, .fromNanoseconds(0), retry.max_delay) - else - fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, charged); - governor.stats.recordRetryDelay(delay); - try governor.io.sleep(delay, .awake); - }, - } - } -} - -/// Runs one exchange on its own unit of concurrency while this task watches -/// the attempt's progress, and cancels it after `Retry.stall_timeout` without -/// progress. The exchange returns a canceled read as `Canceled`, unlogged, -/// so the outcome of a stalled exchange is decided here, as a timeout. -/// Without a unit of concurrency the exchange runs unwatched. -fn watched( - comptime T: type, - context: anytype, - comptime exchangeFn: fn (@TypeOf(context), Attempt) anyerror!Outcome(T), - attempt: Attempt, -) anyerror!Outcome(T) { - const Run = struct { - result: anyerror!Outcome(T) = undefined, - done: std.Io.Event = .unset, - - fn run(self: *@This(), io: std.Io, context_: @TypeOf(context), attempt_: Attempt) void { - self.result = exchangeFn(context_, attempt_); - self.done.set(io); - } - }; - const governor = attempt.governor; - var run: Run = .{}; - var future = governor.io.concurrent(Run.run, .{ &run, governor.io, context, attempt }) catch |err| switch (err) { - error.ConcurrencyUnavailable => return exchangeFn(context, attempt), - }; - const stall_timeout = governor.retry.stall_timeout; - const check: std.Io.Timeout = .{ .duration = .{ - .raw = .fromNanoseconds(@divTrunc(stall_timeout.nanoseconds, stall_checks)), - .clock = .awake, - } }; - var seen = attempt.progress.load(.monotonic); - var quiet: usize = 0; - while (quiet < stall_checks) { - run.done.waitTimeout(governor.io, check) catch |err| switch (err) { - error.Timeout => { - const now = attempt.progress.load(.monotonic); - quiet = if (now == seen) quiet + 1 else 0; - seen = now; - continue; - }, - error.Canceled => { - future.cancel(governor.io); - // An exchange already past its last cancelation point - // finished: its value is the caller's, and the cancel - // reaches the caller at its next cancelation point. - if (run.result) |outcome| switch (outcome) { - .done => { - governor.io.recancel(); - return outcome; - }, - .retry => {}, - } else |_| {} - return error.Canceled; - }, - }; - future.await(governor.io); - return run.result; - } - future.cancel(governor.io); - // It may have finished while being canceled. - if (run.result) |outcome| switch (outcome) { - .done => return outcome, - .retry => {}, - } else |_| {} - log.warn("{s}: {s} received nothing for {d:.1}s; retrying", .{ governor.name, attempt.spec.target, @as(f64, @floatFromInt(stall_timeout.nanoseconds)) / std.time.ns_per_s }); - return .{ .retry = .{ .failure = .timeout } }; -} - -/// Waits out the scope's hold. Returns at once when nothing is held; fails -/// with `error.RateLimited` when the episode outlived its budget. Waiters -/// wake at the deadline plus their own jitter, so the readers of one backend -/// do not all fire in the same instant, and re-check, since the hold may have -/// been extended while they slept. -fn admit(governor: *Governor, spec: RequestSpec) error{ Canceled, RateLimited }!void { - while (true) { - const deadline = deadline: { - governor.mutex.lockUncancelable(governor.io); - defer governor.mutex.unlock(governor.io); - const hold = governor.holdFor(spec.key); - const now: std.Io.Timestamp = .now(governor.io, .awake); - refresh(hold, now); - if (hold.exhausted) return error.RateLimited; - const until = hold.until orelse return; - if (until.nanoseconds <= now.nanoseconds) return; - break :deadline until; - }; - const jitter = fullJitterDelay(governor.io, governor.retry.initial_delay, governor.retry.initial_delay, 0); - const started: std.Io.Timestamp = .now(governor.io, .awake); - try std.Io.Clock.Timestamp.wait(deadline.addDuration(jitter).withClock(.awake), governor.io); - governor.stats.recordHoldWait(started.durationTo(.now(governor.io, .awake))); - } -} - -/// Arms or extends the scope's hold after a throttle. `named` is the delay -/// the server asked for, if any; it is honoured up to `max_hold` and never -/// below `initial_delay`, because `Retry-After: 0` and Hugging Face's -/// `RateLimit; t=0` parse to zero, and a throttle charges no retry: a zero -/// hold would be a hot loop. -fn reportThrottle(governor: *Governor, spec: RequestSpec, named: ?std.Io.Duration) error{RateLimited}!void { - const retry = governor.retry; - governor.mutex.lockUncancelable(governor.io); - defer governor.mutex.unlock(governor.io); - const hold = governor.holdFor(spec.key); - const now: std.Io.Timestamp = .now(governor.io, .awake); - refresh(hold, now); - - if (hold.episode_started) |started| { - if (started.durationTo(now).nanoseconds > retry.throttle_budget.nanoseconds) { - hold.exhausted = true; - // The caller reports the failure; this is the one line that - // says why its request never ran. - log.warn("{s}: rate limited for longer than the throttle budget while reading {s}; giving up", .{ governor.name, spec.target }); - return error.RateLimited; - } - } else { - hold.episode_started = now; - } - - const holding = if (hold.until) |until| now.nanoseconds < until.nanoseconds else false; - if (!holding) { - hold.consecutive += 1; - const backoff = fullJitterDelay(governor.io, retry.initial_delay, retry.max_delay, hold.consecutive - 1); - var delay = backoff; - if (named) |value| { - if (value.nanoseconds > delay.nanoseconds) delay = value; - } - hold.length = clampDelay(delay, retry.initial_delay, retry.max_hold); - hold.until = now.addDuration(hold.length); - log.warn("{s}: {s} is rate limiting; holding its requests for {d:.3}s", .{ - governor.name, - spec.target, - @as(f64, @floatFromInt(hold.length.toNanoseconds())) / std.time.ns_per_s, - }); - governor.stats.recordHold(); - return; - } - // A request already in flight when the hold armed got its own answer, - // with a later reset: extend, never shorten. - const asked = named orelse return; - const capped = clampDelay(asked, retry.initial_delay, retry.max_hold); - const extended = now.addDuration(capped); - if (extended.nanoseconds > hold.until.?.nanoseconds) { - hold.until = extended; - hold.length = capped; - } -} - -/// A clean window as long as the hold itself, with no new throttle, ends the -/// episode: nothing else needs counting, since a throttle inside the window -/// would have re-armed `until`. -fn refresh(hold: *Hold, now: std.Io.Timestamp) void { - const until = hold.until orelse return; - if (now.nanoseconds < until.addDuration(hold.length).nanoseconds) return; - hold.* = .{}; -} - -fn clampDelay(value: std.Io.Duration, low: std.Io.Duration, high: std.Io.Duration) std.Io.Duration { - return .fromNanoseconds(std.math.clamp(value.nanoseconds, low.nanoseconds, high.nanoseconds)); -} - -/// The hops `Redirects.follow` takes: std's default. -pub const max_redirects = 3; - -/// What happens to a redirect: the client chases it (`max_redirects` hops), -/// `consume` receives it because `HeadChain` walks the chain itself, or it -/// fails the request. -pub const Redirects = enum { - follow, - surface, - forbid, - - fn behavior(self: Redirects) std.http.Client.Request.RedirectBehavior { - return switch (self) { - .follow => .init(max_redirects), - .surface => .unhandled, - .forbid => .not_allowed, - }; - } -}; - -pub const ExchangeOptions = struct { - method: std.http.Method = .GET, - authorization: std.http.Client.Request.Headers.Value = .default, - content_type: std.http.Client.Request.Headers.Value = .default, - extra_headers: []const std.http.Header = &.{}, - /// Sent as the whole request body; the method must allow one. The - /// writer uses it as its own buffer, so it must be mutable. - payload: ?[]u8 = null, - /// An open, stat or access: an absent or denied object fails the - /// request with its `OpenError` before `consume`, unlogged. - open: bool = false, - redirects: Redirects = .forbid, - /// A refusal that fails the request is logged with up to 1 KiB of its - /// body, where an OAuth endpoint says why, instead of with its head. - log_refusal_body: bool = false, -}; - -/// One HTTP exchange of `attempt`'s request: send, receive the head, -/// classify it, and hand a 2xx (or a redirect `Redirects.surface` hands -/// back) to `consume` while the request is alive. Retryable connection, -/// send and receive failures and every classified status become `.retry`; -/// an unclassified status or any other connect or head failure fails with -/// `error.RequestFailed`, logged at err, except that an open request -/// (`ExchangeOptions.open`) fails an absent or denied object with -/// `FileNotFound` or `PermissionDenied` first, unlogged. The exchange alone -/// is watched (`Retry.stall_timeout`); a governed request its hook runs -/// first (a re-resolve, a token refresh) is watched by its own exchange. -pub fn exchange( - comptime T: type, - attempt: Attempt, - uri: std.Uri, - options: ExchangeOptions, - context: anytype, - comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, -) anyerror!Outcome(T) { - const Exchange = struct { - uri: std.Uri, - options: ExchangeOptions, - context: @TypeOf(context), - - fn run(self: @This(), current: Attempt) anyerror!Outcome(T) { - return roundTrip(T, current, self.uri, self.options, self.context, consume); - } - }; - return watched(T, Exchange{ .uri = uri, .options = options, .context = context }, Exchange.run, attempt); -} - -fn roundTrip( - comptime T: type, - attempt: Attempt, - uri: std.Uri, - options: ExchangeOptions, - context: anytype, - comptime consume: fn (@TypeOf(context), *std.http.Client.Response) anyerror!T, -) anyerror!Outcome(T) { - const spec = attempt.spec; - // A followed redirect keeps each hop's Location and merged path here, - // 8 KiB of each, as much as a `HeadChain` URL; `req.uri` points into it. - var redirect_buffer: [2 * max_redirects * 8 * 1024]u8 = undefined; - var req = attempt.governor.client.request(options.method, uri, .{ - .redirect_behavior = options.redirects.behavior(), - .headers = .{ - // Range offsets and Content-Length count the stored bytes. - .accept_encoding = .{ .override = "identity" }, - .authorization = options.authorization, - .content_type = options.content_type, - }, - .extra_headers = options.extra_headers, - }) catch |err| switch (err) { - error.Timeout => return retryable(T, attempt, "connect", err, .timeout), - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.NameServerFailure, - => return retryable(T, attempt, "connect", err, .transient), - // A stalled exchange that `watched` canceled, or the caller's cancel. - error.Canceled => return err, - error.OutOfMemory => return err, - else => return fatal(attempt, "connect", err), - }; - defer req.deinit(); - - // A request that was not sent whole leaves its connection unusable; - // `deinit` would otherwise pool it, its reader still `ready`. - if (options.payload) |payload| { - req.sendBodyComplete(payload) catch |err| switch (err) { - error.WriteFailed => { - req.connection.?.closing = true; - if (canceled(&req)) return error.Canceled; - return retryable(T, attempt, "send body", err, .transient); - }, - }; - } else { - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - req.connection.?.closing = true; - if (canceled(&req)) return error.Canceled; - return retryable(T, attempt, "send headers", err, .transient); - }, - }; - } - - var res = req.receiveHead(if (options.redirects == .follow) &redirect_buffer else &.{}) catch |err| { - // A failed read already closed it; a redirect re-sends on a new - // connection that a failure leaves unusable too (or null). - if (req.connection) |connection| connection.closing = true; - if (canceled(&req)) return error.Canceled; - switch (err) { - error.Timeout => return retryable(T, attempt, "receive headers", err, .timeout), - error.HttpConnectionClosing, - error.HttpRequestTruncated, - error.ReadFailed, - error.WriteFailed, - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.NameServerFailure, - => return retryable(T, attempt, "receive headers", err, .transient), - // A stalled exchange that `watched` canceled, or the caller's cancel. - error.Canceled => return err, - error.OutOfMemory => return err, - else => return fatal(attempt, "receive headers", err), - } - }; - - if (options.open) { - if (openStatus(res.head.status)) |err| return err; - } - const class = res.head.status.class(); - if (class != .success and !(class == .redirect and options.redirects == .surface)) { - for (spec.retry_once) |status| { - if (status != res.head.status) continue; - log.warn("{s}: {s} answered {d}; retrying once with a fresh request", .{ - attempt.governor.name, - spec.target, - @backingInt(res.head.status), - }); - return .{ .retry = .{ .failure = .transient, .status = res.head.status } }; - } - const failure = classifyStatus(res.head.status, attempt.governor.unavailable) orelse { - if (!options.log_refusal_body) { - log.err("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); - return error.RequestFailed; - } - // Reading the body invalidates the head's strings. - const status = res.head.status; - var body_buffer: [1024]u8 = undefined; - var body: std.Io.Writer = .fixed(&body_buffer); - // A longer body is cut, one that fails is logged as it arrived. - _ = res.reader(&.{}).streamRemaining(&body) catch { - if (canceled(&req)) return error.Canceled; - }; - // One line: an OAuth endpoint spreads its JSON over several. - std.mem.replaceScalar(u8, body.buffered(), '\n', ' '); - log.err("{s}: {s} answered {d}: {s}", .{ attempt.governor.name, spec.target, @backingInt(status), body.buffered() }); - return error.RequestFailed; - }; - log.warn("{s}: {s} failed: {s}", .{ attempt.governor.name, spec.target, res.head.bytes }); - return .{ .retry = .{ - .failure = failure, - .delay = serverRetryDelay(res.head), - .status = res.head.status, - } }; - } - - return .{ .done = consume(context, &res) catch |err| switch (err) { - error.EndOfStream, error.ReadFailed => { - if (canceled(&req)) return error.Canceled; - return retryable(T, attempt, "read body", err, .transient); - }, - else => return err, - } }; -} - -/// Whether the exchange's read or write failed because it was canceled: -/// std's stream keeps the cancel and reports a failed read or write. -fn canceled(req: *const std.http.Client.Request) bool { - const connection = req.connection orelse return false; - if (connection.stream_reader.err) |err| return err == error.Canceled; - if (connection.stream_writer.err) |err| return err == error.Canceled; - return false; -} - -fn retryable( - comptime T: type, - attempt: Attempt, - stage: []const u8, - err: anyerror, - failure: ReadFailure, -) Outcome(T) { - log.warn("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); - return .{ .retry = .{ .failure = failure } }; -} - -fn fatal(attempt: Attempt, stage: []const u8, err: anyerror) error{RequestFailed} { - log.err("{s}: {s} for {s} failed: {}", .{ attempt.governor.name, stage, attempt.spec.target, err }); - return error.RequestFailed; -} - -/// The URLs of a HEAD redirect chain walked by hand: std hands a HEAD its -/// redirect unfollowed. Three buffers in rotation: the current URL, the -/// `Location` it answered (copied out of a head that dies with the -/// response) with the path merged against it, and the resolved next URL. -pub const HeadChain = struct { - urls: [3][8 * 1024]u8 = undefined, - - pub const End = struct { - /// `start_url` itself, or a URL in the chain's buffers. - url: []const u8, - /// Its `Content-Length`, when it sent one. - size: ?u64, - }; - - /// Follows `start_url` through at most `max_hops` redirects, absolute - /// or relative, to the URL that answers a 2xx. Each hop is one governed - /// HEAD, so a server that rate limits it holds the backend as a - /// throttled GET would. `authorize` names the credentials each URL is - /// sent. An absent or denied object fails the walk - /// (`ExchangeOptions.open`). - pub fn resolve( - self: *HeadChain, - governor: *Governor, - target: []const u8, - start_url: []const u8, - max_hops: usize, - context: anytype, - comptime authorize: fn (@TypeOf(context), url: []const u8) std.http.Client.Request.Headers.Value, - ) !End { - const Hop = struct { - context: @TypeOf(context), - url: []const u8, - uri: std.Uri, - /// Where a redirect's `Location` is copied. - location: []u8, - - const Result = union(enum) { end: ?u64, redirect: usize }; - - fn attempt(hop: *@This(), current: Attempt) anyerror!Outcome(Result) { - return exchange(Result, current, hop.uri, .{ - .method = .HEAD, - .authorization = authorize(hop.context, hop.url), - .open = true, - .redirects = .surface, - }, hop, consume); - } - - fn consume(hop: *@This(), res: *std.http.Client.Response) anyerror!Result { - if (res.head.status.class() == .success) return .{ .end = res.head.content_length }; - const location = res.head.location orelse return error.HttpRedirectLocationMissing; - if (location.len > hop.location.len) return error.HttpRedirectLocationOversize; - @memcpy(hop.location[0..location.len], location); - return .{ .redirect = location.len }; - } - }; - - var url = start_url; - // `start_url` is in none of the buffers. - var current: usize = self.urls.len - 1; - var hops: usize = 0; - while (true) { - const uri: std.Uri = try .parse(url); - const scratch = (current + 1) % self.urls.len; - var hop: Hop = .{ .context = context, .url = url, .uri = uri, .location = &self.urls[scratch] }; - const location_len = switch (try perform(Hop.Result, governor, .{ .target = target, .key = authorityOf(uri) }, &hop, Hop.attempt)) { - .end => |size| return .{ .url = url, .size = size }, - .redirect => |len| len, - }; - - if (hops == max_hops) return error.TooManyHttpRedirects; - hops += 1; - const output = (current + 2) % self.urls.len; - var aux: []u8 = &self.urls[scratch]; - const resolved = try uri.resolveInPlace(location_len, &aux); - url = try std.fmt.bufPrint(&self.urls[output], "{f}", .{resolved}); - current = output; - } - } -}; - -/// Retry classification of a non-2xx status; null when the status is not -/// retried. `unavailable` is what a 503 means for the backend: rate limiting -/// on the object stores (AWS `SlowDown`, GCS), a server failure elsewhere. -fn classifyStatus(status: std.http.Status, unavailable: ReadFailure) ?ReadFailure { - return switch (status) { - .request_timeout => .timeout, - .too_many_requests => .throttle, - .service_unavailable => unavailable, - else => if (status.class() == .server_error) .server_failure else null, - }; -} - -/// What an open, stat or access of a remote path fails with (vfs.zig's -/// header); `openStatus` maps the statuses, and anything else is logged -/// once, by `exchange` or `Governor.openFailure`. -pub const OpenError = error{ FileNotFound, PermissionDenied, Canceled, SystemResources, Unexpected }; - -/// The error a status means for the path an open request -/// (`ExchangeOptions.open`) asked about; null when the status is not one. -fn openStatus(status: std.http.Status) ?error{ FileNotFound, PermissionDenied } { - return switch (status) { - .not_found, .gone => error.FileNotFound, - .unauthorized, .forbidden => error.PermissionDenied, - else => null, - }; -} - -/// The retry delay the server names, if any: `Retry-After` delta-seconds -/// (the HTTP-date form is not parsed and falls back to the jittered delay) -/// or the `t=` reset of the `RateLimit` header Hugging Face sends. -fn serverRetryDelay(head: std.http.Client.Response.Head) ?std.Io.Duration { - var it = head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Retry-After")) { - return delaySeconds(header.value) orelse continue; - } - if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { - var parts = std.mem.splitScalar(u8, header.value, ';'); - while (parts.next()) |part| { - const trimmed = std.mem.trim(u8, part, " \t"); - if (std.mem.startsWith(u8, trimmed, "t=")) return delaySeconds(trimmed[2..]) orelse continue; - } - } - } - return null; -} - -fn delaySeconds(value: []const u8) ?std.Io.Duration { - const seconds = std.fmt.parseInt(u32, std.mem.trim(u8, value, " \t"), 10) catch return null; - return .fromSeconds(seconds); -} - -fn fullJitterDelay( - io: std.Io, - initial: std.Io.Duration, - maximum: std.Io.Duration, - attempt: usize, -) std.Io.Duration { - const max_delay_ns: i96 = @min( - maximum.toNanoseconds(), - initial.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), - ); - if (max_delay_ns <= 0) return .fromNanoseconds(0); - - var seed: u64 = undefined; - io.random(@ptrCast(&seed)); - var prng: std.Random.DefaultPrng = .init(seed); - return .fromNanoseconds(prng.random().intRangeAtMost(i96, 0, max_delay_ns)); -} - -const test_spec: RequestSpec = .{ .target = "object" }; - -fn testGovernor(initial_ms: i64, max_delay_ms: i64, max_hold_ms: i64, budget_ms: i64) Governor { - // No attempt in these tests reaches the network: no client. - return .init(std.testing.io, undefined, "test", .throttle, .{ - .max_retries = 3, - .initial_delay = .fromMilliseconds(initial_ms), - .max_delay = .fromMilliseconds(max_delay_ms), - .max_hold = .fromMilliseconds(max_hold_ms), - .throttle_budget = .fromMilliseconds(budget_ms), - }); -} - -test "a throttle arms a hold of at least the initial delay, even when the server names zero" { - var governor = testGovernor(20, 20, 1000, 300_000); - - // `Retry-After: 0` and Hugging Face's `RateLimit; t=0` parse to zero. - try reportThrottle(&governor, test_spec, .fromSeconds(0)); - const hold = governor.hold; - try std.testing.expect(hold.until != null); - try std.testing.expectEqual(@as(i96, 20 * std.time.ns_per_ms), hold.length.nanoseconds); - try std.testing.expectEqual(@as(usize, 1), hold.consecutive); - try std.testing.expect(hold.episode_started != null); - try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); -} - -test "a longer named delay extends a running hold and a shorter one does not" { - var governor = testGovernor(10, 10, 1000, 300_000); - - try reportThrottle(&governor, test_spec, null); - const armed = governor.hold.until.?; - - // A request that was already in flight got its own answer, with a later - // reset: the hold extends. - try reportThrottle(&governor, test_spec, .fromMilliseconds(500)); - const extended = governor.hold.until.?; - try std.testing.expect(extended.nanoseconds > armed.nanoseconds); - - // A shorter one never shortens it, and neither arms a second hold. - try reportThrottle(&governor, test_spec, .fromMilliseconds(1)); - try std.testing.expectEqual(extended.nanoseconds, governor.hold.until.?.nanoseconds); - try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); - try std.testing.expectEqual(@as(u64, 1), governor.stats.holds.load(.acquire)); -} - -test "a named delay is capped by max_hold" { - var governor = testGovernor(10, 10, 50, 300_000); - - try reportThrottle(&governor, test_spec, .fromSeconds(3600)); - try std.testing.expectEqual(@as(i96, 50 * std.time.ns_per_ms), governor.hold.length.nanoseconds); -} - -test "a second throttle inside the clean window doubles the backoff base" { - var governor = testGovernor(10, 1000, 10_000, 300_000); - - try reportThrottle(&governor, test_spec, null); - try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); - - // The hold has passed but the clean window has not: rewind it by hand - // rather than sleeping through a race. - const now: std.Io.Timestamp = .now(std.testing.io, .awake); - governor.hold.until = now.subDuration(.fromMilliseconds(1)); - try reportThrottle(&governor, test_spec, null); - try std.testing.expectEqual(@as(usize, 2), governor.hold.consecutive); - // Full jitter over twice the base, floored at the initial delay. - try std.testing.expect(governor.hold.length.nanoseconds >= 10 * std.time.ns_per_ms); - try std.testing.expect(governor.hold.length.nanoseconds <= 20 * std.time.ns_per_ms); - try std.testing.expectEqual(@as(u64, 2), governor.stats.holds.load(.acquire)); -} - -test "a clean window as long as the hold ends the episode" { - var governor = testGovernor(10, 1000, 10_000, 300_000); - - try reportThrottle(&governor, test_spec, null); - const length = governor.hold.length; - const now: std.Io.Timestamp = .now(std.testing.io, .awake); - governor.hold.until = now.subDuration(length).subDuration(.fromMilliseconds(1)); - - try reportThrottle(&governor, test_spec, null); - try std.testing.expectEqual(@as(usize, 1), governor.hold.consecutive); - try std.testing.expectEqual(@as(i96, 10 * std.time.ns_per_ms), governor.hold.length.nanoseconds); -} - -test "an episode longer than the throttle budget fails requests until a clean window" { - var governor = testGovernor(10, 10, 10_000, 50); - - try reportThrottle(&governor, test_spec, .fromSeconds(30)); - const now: std.Io.Timestamp = .now(std.testing.io, .awake); - governor.hold.episode_started = now.subDuration(.fromMilliseconds(51)); - - try std.testing.expectError(error.RateLimited, reportThrottle(&governor, test_spec, null)); - try std.testing.expect(governor.hold.exhausted); - // Every other request of this backend fails at once, without waiting the - // hold out. - try std.testing.expectError(error.RateLimited, admit(&governor, test_spec)); - - // The next clean window clears the episode. - governor.hold.until = now.subDuration(governor.hold.length).subDuration(.fromMilliseconds(1)); - try admit(&governor, test_spec); - try std.testing.expect(governor.hold.until == null); - try std.testing.expect(!governor.hold.exhausted); -} - -test "admit returns at once when nothing is held" { - var governor = testGovernor(10, 10, 1000, 300_000); - - const started: std.Io.Timestamp = .now(std.testing.io, .awake); - try admit(&governor, test_spec); - try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds < 5 * std.time.ns_per_ms); - try std.testing.expectEqual(@as(u64, 0), governor.stats.hold_wait_ns.load(.acquire)); -} - -test "admit waits a hold out and counts the wait" { - var governor = testGovernor(1, 1, 1000, 300_000); - - try reportThrottle(&governor, test_spec, .fromMilliseconds(15)); - const started: std.Io.Timestamp = .now(std.testing.io, .awake); - try admit(&governor, test_spec); - try std.testing.expect(started.durationTo(.now(std.testing.io, .awake)).nanoseconds >= 10 * std.time.ns_per_ms); - try std.testing.expect(governor.stats.hold_wait_ns.load(.acquire) > 0); -} - -test "a waiter blocked on a hold is cancelable" { - const io = std.testing.io; - var governor = testGovernor(1, 1, 100_000, 300_000); - try reportThrottle(&governor, test_spec, .fromSeconds(60)); - - var result: std.atomic.Value(u16) = .init(0); - var started: std.Io.Event = .unset; - var group: std.Io.Group = .init; - try group.concurrent(io, struct { - fn run(governor_: *Governor, started_: *std.Io.Event, result_: *std.atomic.Value(u16)) void { - started_.set(governor_.io); - admit(governor_, test_spec) catch |err| { - result_.store(@intFromError(err), .release); - return; - }; - } - }.run, .{ &governor, &started, &result }); - try started.wait(io); - try io.sleep(.fromMilliseconds(5), .awake); - group.cancel(io); - try std.testing.expectEqual(@intFromError(error.Canceled), result.load(.acquire)); -} - -test "only rate limiting holds: a timeout and a server failure keep the per-request budget" { - var governor = testGovernor(0, 0, 1000, 300_000); - - const Attempts = struct { - seen: usize = 0, - fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { - self.seen += 1; - return switch (self.seen) { - 1 => .{ .retry = .{ .failure = .timeout } }, - 2 => .{ .retry = .{ .failure = .server_failure } }, - else => .{ .done = self.seen }, - }; - } - }; - var attempts: Attempts = .{}; - try std.testing.expectEqual(@as(usize, 3), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); - try std.testing.expect(governor.hold.until == null); - try std.testing.expectEqual(@as(u64, 0), governor.stats.holds.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), governor.stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(u64, 3), governor.stats.physical_requests.load(.acquire)); -} - -test "a throttle retries without charging the retry budget" { - var governor = testGovernor(0, 0, 1000, 300_000); - - const Attempts = struct { - seen: usize = 0, - fn run(self: *@This(), current: Attempt) anyerror!Outcome(usize) { - self.seen += 1; - // The retry budget is three; ten throttles do not exhaust it. - if (self.seen <= 10) return .{ .retry = .{ .failure = .throttle, .status = .too_many_requests } }; - try std.testing.expectEqual(@as(?std.http.Status, .too_many_requests), current.previous_status); - return .{ .done = self.seen }; - } - }; - var attempts: Attempts = .{}; - try std.testing.expectEqual(@as(usize, 11), try perform(usize, &governor, test_spec, &attempts, Attempts.run)); - try std.testing.expectEqual(@as(u64, 0), governor.stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(u64, 10), governor.stats.throttles.load(.acquire)); - try std.testing.expectEqual(@as(u64, 11), governor.stats.physical_requests.load(.acquire)); - try std.testing.expect(governor.stats.holds.load(.acquire) >= 1); -} - -test "the retry budget still bounds failures that are not rate limiting" { - var governor = testGovernor(0, 0, 1000, 300_000); - - const Attempts = struct { - seen: usize = 0, - fn run(self: *@This(), _: Attempt) anyerror!Outcome(usize) { - self.seen += 1; - return .{ .retry = .{ .failure = .transient } }; - } - }; - var attempts: Attempts = .{}; - try std.testing.expectError(error.RetriesExhausted, perform(usize, &governor, test_spec, &attempts, Attempts.run)); - // The first attempt plus `max_retries` of them. - try std.testing.expectEqual(@as(usize, 4), attempts.seen); -} - -test "retry status classification is typed and 503 depends on the backend" { - try std.testing.expectEqual(ReadFailure.timeout, classifyStatus(.request_timeout, .server_failure).?); - try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.too_many_requests, .server_failure).?); - try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.bad_gateway, .throttle).?); - try std.testing.expectEqual(ReadFailure.server_failure, classifyStatus(.service_unavailable, .server_failure).?); - try std.testing.expectEqual(ReadFailure.throttle, classifyStatus(.service_unavailable, .throttle).?); - try std.testing.expect(classifyStatus(.not_found, .throttle) == null); -} - -test "absent and denied statuses map to one error each" { - try std.testing.expect(openStatus(.not_found).? == error.FileNotFound); - try std.testing.expect(openStatus(.gone).? == error.FileNotFound); - try std.testing.expect(openStatus(.unauthorized).? == error.PermissionDenied); - try std.testing.expect(openStatus(.forbidden).? == error.PermissionDenied); - for ([_]std.http.Status{ .ok, .partial_content, .bad_request, .too_many_requests, .internal_server_error, .service_unavailable }) |status| { - try std.testing.expect(openStatus(status) == null); - } - - const governor = testGovernor(10, 10, 1000, 300_000); - try std.testing.expect(governor.openFailure("object", error.FileNotFound) == error.FileNotFound); - try std.testing.expect(governor.openFailure("object", error.PermissionDenied) == error.PermissionDenied); - try std.testing.expect(governor.openFailure("object", error.Canceled) == error.Canceled); - try std.testing.expect(governor.openFailure("object", error.OutOfMemory) == error.SystemResources); - try std.testing.expect(governor.openFailure("object", error.RequestFailed) == error.Unexpected); - try std.testing.expect(governor.openFailure("object", error.InvalidContentRange) == error.Unexpected); -} - -test "a failed read is Canceled or Unexpected" { - const governor = testGovernor(10, 10, 1000, 300_000); - try std.testing.expect(governor.readFailure("object", 0, error.Canceled) == error.Canceled); - try std.testing.expect(governor.readFailure("object", 0, error.RequestFailed) == error.Unexpected); - try std.testing.expect(governor.readFailure("object", 0, error.InvalidContentRange) == error.Unexpected); -} - -test "server retry delay comes from Retry-After seconds or a RateLimit reset" { - const Head = std.http.Client.Response.Head; - const retry_after = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nretry-after: 3\r\n\r\n"); - try std.testing.expectEqual(std.Io.Duration.fromSeconds(3), serverRetryDelay(retry_after).?); - - const rate_limit = try Head.parse("HTTP/1.1 429 Too Many Requests\r\nRateLimit: \"default\"; r=0; t=7\r\n\r\n"); - try std.testing.expectEqual(std.Io.Duration.fromSeconds(7), serverRetryDelay(rate_limit).?); - - const http_date = try Head.parse("HTTP/1.1 503 Service Unavailable\r\nRetry-After: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n"); - try std.testing.expect(serverRetryDelay(http_date) == null); - const none = try Head.parse("HTTP/1.1 500 Internal Server Error\r\n\r\n"); - try std.testing.expect(serverRetryDelay(none) == null); -} - -test "a server-named retry delay sleeps at most max_delay" { - var governor = testGovernor(1, 20, 20, 1000); - const Failing = struct { - seen: usize = 0, - fn attempt(self: *@This(), _: Attempt) anyerror!Outcome(usize) { - self.seen += 1; - // A failing server that asks for an hour. - if (self.seen == 1) return .{ .retry = .{ .failure = .server_failure, .delay = .fromSeconds(3600) } }; - return .{ .done = self.seen }; - } - }; - var failing: Failing = .{}; - try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &failing, Failing.attempt)); - try std.testing.expectEqual(@as(u64, 20 * std.time.ns_per_ms), governor.stats.retry_delay_ns.load(.acquire)); -} - -test "an attempt that receives nothing is canceled and retried as a timeout" { - var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ - .max_retries = 3, - .initial_delay = .fromMilliseconds(1), - .max_delay = .fromMilliseconds(1), - .max_hold = .fromMilliseconds(1), - .throttle_budget = .fromSeconds(1), - .stall_timeout = .fromMilliseconds(120), - }); - const Hang = struct { - seen: usize = 0, - progressed: u64 = 0, - // Watched as `exchange` watches its round trip. - fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { - return watched(usize, self, receive, current); - } - - fn receive(self: *@This(), current: Attempt) anyerror!Outcome(usize) { - self.seen += 1; - // The first exchange hangs, the second one keeps receiving and - // takes longer than the stall timeout overall. - if (self.seen == 1) { - try std.testing.io.sleep(.fromSeconds(60), .awake); - return error.TestUnexpectedResult; - } - for (0..6) |_| { - try std.testing.io.sleep(.fromMilliseconds(40), .awake); - _ = current.progress.fetchAdd(1, .monotonic); - self.progressed += 1; - } - return .{ .done = self.seen }; - } - }; - var hang: Hang = .{}; - try std.testing.expectEqual(@as(usize, 2), try perform(usize, &governor, test_spec, &hang, Hang.attempt)); - try std.testing.expectEqual(@as(u64, 6), hang.progressed); - try std.testing.expectEqual(@as(u64, 1), governor.stats.timeouts.load(.acquire)); -} - -test "an attempt waiting on a request of its own is not a stall" { - var governor: Governor = .init(std.testing.io, undefined, "test", .throttle, .{ - .max_retries = 3, - .initial_delay = .fromMilliseconds(1), - .max_delay = .fromMilliseconds(1), - .max_hold = .fromMilliseconds(1), - .throttle_budget = .fromSeconds(1), - .stall_timeout = .fromMilliseconds(120), - }); - const Nested = struct { - fn attempt(self: *@This(), current: Attempt) anyerror!Outcome(usize) { - _ = try perform(usize, current.governor, test_spec, self, wait); - return .{ .done = 1 }; - } - - // A re-resolve or token refresh waiting out a hold, outside any - // exchange, for longer than the stall timeout. - fn wait(_: *@This(), _: Attempt) anyerror!Outcome(usize) { - try std.testing.io.sleep(.fromMilliseconds(300), .awake); - return .{ .done = 0 }; - } - }; - var nested: Nested = .{}; - try std.testing.expectEqual(@as(usize, 1), try perform(usize, &governor, test_spec, &nested, Nested.attempt)); - try std.testing.expectEqual(@as(u64, 0), governor.stats.timeouts.load(.acquire)); - try std.testing.expectEqual(@as(u64, 2), governor.stats.physical_requests.load(.acquire)); -} - -test "an exchange canceled while it waits for the head returns Canceled, not a retry" { - const MockServer = @import("mock_server.zig").MockServer; - const io = std.testing.io; - - // The GET waits unanswered for a second one that never comes. - var server = try MockServer.init(io, "object", .{ .barrier_gets = 2 }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = std.testing.allocator, .io = io }; - defer client.deinit(); - var governor: Governor = .init(io, &client, "test", .throttle, .{}); - var url_buffer: [64]u8 = undefined; - const uri: std.Uri = try .parse(try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()})); - - const Exchange = struct { - // Unwatched, so that the cancel lands in the read rather than in `watched`. - fn run(governor_: *Governor, uri_: std.Uri, result: *anyerror!Outcome(void)) void { - var progress: std.atomic.Value(u64) = .init(0); - result.* = roundTrip(void, .{ .governor = governor_, .spec = test_spec, .progress = &progress }, uri_, .{ - .extra_headers = &.{.{ .name = "Range", .value = "bytes=0-5" }}, - }, {}, consume); - } - - fn consume(_: void, _: *std.http.Client.Response) anyerror!void {} - }; - var result: anyerror!Outcome(void) = .{ .done = {} }; - var group: std.Io.Group = .init; - try group.concurrent(io, Exchange.run, .{ &governor, uri, &result }); - try io.sleep(.fromMilliseconds(50), .awake); - group.cancel(io); - try std.testing.expectError(error.Canceled, result); - - try server.stop(io); -} - -test "the request authority is the rate-limit scope" { - const uri: std.Uri = try .parse("https://huggingface.co/api/models/x"); - try std.testing.expectEqualStrings("huggingface.co", authorityOf(uri)); -} diff --git a/vfs/s3.zig b/vfs/s3.zig index 65132a64..eb849e21 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -2,10 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); -const range_read = @import("range_read.zig"); -const request = @import("request.zig"); +const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const Backend = @import("base.zig").Backend; const log = std.log.scoped(.@"zml/vfs/s3"); @@ -143,26 +141,185 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { - /// Without both keys, requests go unsigned. - pub const Options = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *S3, + uri: std.Uri, + url: []const u8, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; + + const signer: AwsSigV4 = .{ + .access_key = job.batch.backend.config.access_key, + .secret_key = job.batch.backend.config.secret_key, + .region = job.batch.backend.config.region, + .service = job.batch.backend.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = signer.generateAuthHeader( + &authorization_buffer, + .GET, + job.batch.uri, + timestamp, + &.{.{ .name = "Range", .value = range_header }}, + ) catch |err| { + log.err("Failed to generate auth header: {}", .{err}); + return err; + }; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; + + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + + pub const Config = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, endpoint_url: []const u8, region: []const u8, - retry: request.Retry = .{}, + auth_service: []const u8 = "s3", }; - /// The strings of the `Options` given to `init`, owned. - const Config = struct { - access_key: ?[]const u8, - secret_key: ?[]const u8, - endpoint_url: []const u8, - region: []const u8, - }; - - /// The SigV4 service every request is signed for. - const auth_service = "s3"; - const Handle = struct { pub const Type = enum { file, directory }; @@ -187,24 +344,38 @@ pub const S3 = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, + client: *std.http.Client, config: Config, - governor: request.Governor, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, options: Options) !S3 { + pub fn init( + allocator: std.mem.Allocator, + inner: std.Io, + http_client: *std.http.Client, + config: Config, + opts: InitOpts, + ) !S3 { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + return .{ .allocator = allocator, .base = .init(inner), - // S3 rate limiting is `503 SlowDown` as well as `429`. - .governor = .init(inner, http_client, "s3", .throttle, options.retry), + .client = http_client, + .read_pool = read_pool, .config = .{ - .access_key = if (options.access_key) |k| try allocator.dupe(u8, k) else null, - .secret_key = if (options.secret_key) |k| try allocator.dupe(u8, k) else null, - .endpoint_url = try allocator.dupe(u8, options.endpoint_url), - .region = try allocator.dupe(u8, options.region), + .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, config.endpoint_url), + .region = try allocator.dupe(u8, config.region), + .auth_service = try allocator.dupe(u8, config.auth_service), }, }; } @@ -251,11 +422,13 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }); + }, .{}); } pub fn deinit(self: *S3) void { - self.governor.logTotals(); + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -282,6 +455,7 @@ pub const S3 = struct { if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); self.allocator.free(self.config.endpoint_url); self.allocator.free(self.config.region); + self.allocator.free(self.config.auth_service); } pub fn io(self: *S3) std.Io { @@ -309,13 +483,6 @@ pub const S3 = struct { }; } - pub fn backend(self: *S3) Backend { - return .{ - .io = self.io(), - .read_hints = .{ .high_latency = true }, - }; - } - fn openHandle(self: *S3) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -365,9 +532,9 @@ pub const S3 = struct { switch (operation) { .file_read_streaming => |o| { const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| switch (self.governor.readFailure(handle.uri, handle.pos, err)) { - error.Canceled => return error.Canceled, - error.Unexpected => return .{ .file_read_streaming = error.InputOutput }, + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { @@ -413,9 +580,11 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; return .{ .inode = 0, @@ -430,19 +599,18 @@ pub const S3 = struct { }; } - fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; - _ = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); - } + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; + var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const size = self.fetchSize(path) catch |err| return self.governor.openFailure(path, err); - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; @@ -468,16 +636,7 @@ pub const S3 = struct { } const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch |err| switch (err) { - error.Canceled => return error.Canceled, - error.OutOfMemory => return error.SystemResources, - // Logged by `exchange`. - error.RequestFailed => return error.Unexpected, - else => { - log.err("listing {s} failed: {t}", .{ handle.uri, err }); - return error.Unexpected; - }, - }; + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; self.dir_read_states.put(self.allocator, reader, .{ .index = 0, @@ -562,7 +721,10 @@ pub const S3 = struct { fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| self.governor.readFailure(handle.uri, offset, err); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { @@ -572,7 +734,7 @@ pub const S3 = struct { if (relative_offset >= 0) { handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); } else { - const abs_offset: u64 = @abs(relative_offset); + const abs_offset: u64 = @intCast(-relative_offset); if (abs_offset > handle.pos) { handle.pos = 0; } else { @@ -658,38 +820,50 @@ pub const S3 = struct { .fragment = null, }; - var listing: Listing = .{ .s3 = self, .uri = uri }; - var target_buffer: [256]u8 = undefined; - return request.perform([]u8, &self.governor, .{ - .target = std.fmt.bufPrint(&target_buffer, "{s}/{s}", .{ bucket, key_prefix }) catch bucket, - .key = request.authorityOf(uri), - }, &listing, Listing.attempt); + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); + + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); + + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); } - /// The bucket listing, signed again on every attempt like every other - /// S3 request (`Signature`). - const Listing = struct { - s3: *S3, - uri: std.Uri, - signature: Signature = .{}, - - fn attempt(self: *Listing, current: request.Attempt) anyerror!request.Outcome([]u8) { - return request.exchange([]u8, current, self.uri, .{ - .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{}), - .extra_headers = &self.signature.headers, - }, self, Listing.consume); - } - - fn consume(self: *Listing, res: *std.http.Client.Response) anyerror![]u8 { - const allocator = self.s3.allocator; - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(allocator, content_len) - else - // When we don't have a content length, put a reasonable limit - try res.reader(&.{}).allocRemaining(allocator, .limited(1024 * 1024)); - } - }; - fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); const body = try self.listObjectsBody(endpoint, bucket, key_prefix); @@ -731,132 +905,93 @@ pub const S3 = struct { return try results.toOwnedSlice(self.allocator); } - fn fetchSize(self: *S3, path: []const u8) !u64 { + fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; var url_buf: [8 * 1024]u8 = undefined; + + const path = try self.resolvePath(dir, sub_path, &path_buffer); const url = try self.s3Url(path, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - var head: SizeRequest = .{ .s3 = self, .uri = uri }; - return request.perform(u64, &self.governor, .{ .target = url, .key = request.authorityOf(uri) }, &head, SizeRequest.attempt); - } + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); - /// The object's size: one signed HEAD (`ExchangeOptions.open`; S3's 403 - /// for a missing key: vfs.zig's header). - const SizeRequest = struct { - s3: *S3, - uri: std.Uri, - signature: Signature = .{}, + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; - fn attempt(self: *SizeRequest, current: request.Attempt) anyerror!request.Outcome(u64) { - return request.exchange(u64, current, self.uri, .{ - .method = .HEAD, - .authorization = try self.signature.sign(self.s3, .HEAD, self.uri, &.{}), - .extra_headers = &self.signature.headers, - .open = true, - }, self, SizeRequest.consume); - } + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); - fn consume(_: *SizeRequest, res: *std.http.Client.Response) anyerror!u64 { - return res.head.content_length orelse error.MissingContentLength; - } - }; - - fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - var url_buf: [8 * 1024]u8 = undefined; - const url = try self.s3Url(handle.uri, &url_buf); - var signed: SignedRequest = .{ .s3 = self, .uri = std.Uri.parse(url) catch return error.BadPathName }; - return range_read.performRangeRead(&self.governor, .{ - .request = .{ .target = url, .key = request.authorityOf(signed.uri) }, - }, &signed, SignedRequest.prepare, data, offset, handle.size); - } - - /// One SigV4-signed GET. The signature covers `x-amz-date` and `Range`, - /// so it is recomputed for every attempt. - const SignedRequest = struct { - s3: *S3, - uri: std.Uri, - signature: Signature = .{}, - - fn prepare(self: *SignedRequest, _: request.Attempt, range: std.http.Header) anyerror!range_read.PreparedRequest { - return .{ - .uri = self.uri, - .authorization = try self.signature.sign(self.s3, .GET, self.uri, &.{range}), - .extra_headers = &self.signature.headers, - }; - } - }; - - /// One attempt's SigV4 headers. Every request signs each attempt anew, - /// since the signature covers `x-amz-date`. The returned authorization - /// and `headers` point into this storage. - const Signature = struct { - timestamp: [16]u8 = undefined, - authorization: [512]u8 = undefined, - headers: [2]std.http.Header = undefined, - - /// The authorization over `uri` and `signed_headers` beside the two - /// headers it sets; omitted without credentials. - fn sign( - self: *Signature, - s3: *S3, - method: std.http.Method, - uri: std.Uri, - signed_headers: []const std.http.Header, - ) !std.http.Client.Request.Headers.Value { - const timestamp = try s3.getTimestamp(&self.timestamp); - const signer: AwsSigV4 = .{ - .access_key = s3.config.access_key, - .secret_key = s3.config.secret_key, - .region = s3.config.region, - .service = auth_service, - }; - const authorization = try signer.generateAuthHeader(&self.authorization, method, uri, timestamp, signed_headers); - self.headers = .{ + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ .{ .name = "x-amz-date", .value = timestamp }, .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + return switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + else => blk: { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + } + + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; + + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); + const uri = std.Uri.parse(url) catch return error.BadPathName; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .url = url, + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, }; - return if (authorization) |value| .{ .override = value } else .omit; } - }; + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } }; - -test "AwsSigV4 signs the HEAD, listing and Range GET that S3 sends" { - // AWS's example keys and date. The expected signatures are botocore's - // (S3SigV4Auth) for the same requests, with UNSIGNED-PAYLOAD as every - // request here is sent. - const signer: AwsSigV4 = .{ - .access_key = "AKIAIOSFODNN7EXAMPLE", - .secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - .region = "us-east-1", - .service = "s3", - }; - const timestamp = "20130524T000000Z"; - const credential = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, "; - var buffer: [512]u8 = undefined; - - const object = try std.Uri.parse("https://s3.amazonaws.com/examplebucket/test.txt"); - try std.testing.expectEqualStrings( - credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d360d9545e619df15fd61e08c5b21c3e2219ee708af3b95ac439b0ee6fdf44b0", - (try signer.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})).?, - ); - try std.testing.expectEqualStrings( - credential ++ "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f483aaef7a23c6af727aeed791410b7f43f0d696542829b6bf6edc1c7cb112c4", - (try signer.generateAuthHeader(&buffer, .GET, object, timestamp, &.{.{ .name = "Range", .value = "bytes=0-9" }})).?, - ); - const listing = try std.Uri.parse("https://s3.amazonaws.com/examplebucket?delimiter=%2F&list-type=2&max-keys=1000&prefix=dir%2F"); - try std.testing.expectEqualStrings( - credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0a8491caa0b04a231c4ea74a87bde51ef60fbe69b496b206f4d07a4c9031188a", - (try signer.generateAuthHeader(&buffer, .GET, listing, timestamp, &.{})).?, - ); - // A custom endpoint signs its port as part of the host. - const minio = try std.Uri.parse("http://127.0.0.1:9000/models/dir/obj.bin"); - try std.testing.expectEqualStrings( - credential ++ "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=84ff2d687bd88454372c3277af863c28ca047204936b346b80bf93456d0870d4", - (try signer.generateAuthHeader(&buffer, .HEAD, minio, timestamp, &.{})).?, - ); - - // Without both keys, requests go unsigned. - const anonymous: AwsSigV4 = .{ .access_key = "AKIAIOSFODNN7EXAMPLE", .secret_key = null, .region = "us-east-1", .service = "s3" }; - try std.testing.expectEqual(null, try anonymous.generateAuthHeader(&buffer, .HEAD, object, timestamp, &.{})); -} diff --git a/vfs/vfs.zig b/vfs/vfs.zig index 8a52e763..fe626ff9 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -1,87 +1,25 @@ -//! A `std.Io` over local files and remote objects. A bare path or a -//! `file://` URI is served by the inner io; a URI such as -//! `hf://org/model/model.safetensors` or `s3://bucket/key` by the backend -//! registered for its scheme (`registerBackend`, or the four of `Remotes`). -//! Readers take `io()` and need not know which they read; one that must -//! (the loader, for read sizes and direct I/O) asks `fromIo`. -//! -//! The VFS, and every backend registered on it, must outlive each file -//! opened through it and each reader that keeps one open: a `Loader` may -//! keep its source files open until its `deinit`. -//! -//! Errors of a remote path, alike on `HTTP`, `HF`, `S3` and `GCS`: opening, -//! stat-ing or probing it (`access`) fails with `FileNotFound` for a missing -//! object (404, 410, or a path absent from an `HF` repository's listing), -//! `PermissionDenied` for a denied one (401, 403), `Canceled` for a canceled -//! task, and `SystemResources` when a request cannot allocate. Any other -//! failure of a request is logged at err with the backend and the path, and -//! surfaces as `Unexpected`. On S3 a principal without `s3:ListBucket` gets -//! 403 for a missing key too, so a probe of an optional object treats both -//! as absent. A positional read that still fails after its retries is -//! logged at err and surfaces as `Unexpected`; a streaming read as -//! `InputOutput`. A throttled backend holds its requests rather than failing -//! them, until its throttle budget runs out (`request.Retry`). A remote path -//! is read-only: creating or deleting a file or a directory there, or -//! opening a file for writing, is logged at err and fails with -//! `ReadOnlyFileSystem`. A local file fails as the inner io does (a denied -//! one is `AccessDenied` there); it may be read past the page cache -//! (`useDirectIo`). Each remote backend logs its read totals at debug from -//! its `deinit`. const std = @import("std"); const stdx = @import("stdx"); +pub const File = @import("file.zig").File; pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; -/// A test-only HTTP origin for tests above this layer (mock_server.zig). -pub const MockServer = @import("mock_server.zig").MockServer; -const base_module = @import("base.zig"); -const direct_io = @import("direct_io.zig"); -pub const Backend = base_module.Backend; -pub const ReadHints = base_module.ReadHints; -const VFSBase = base_module.VFSBase; -/// See `useDirectIo`. -pub const DirectIo = direct_io.Policy; -/// What every read of a direct file must meet in offset, buffers and total -/// (`useDirectIo`); 0 on a system without direct I/O, where no file is. -pub const direct_io_alignment: usize = if (direct_io.supported) direct_io.alignment else 0; - -test { - _ = @import("gcs.zig"); - _ = @import("http_acceptance_test.zig"); -} +pub const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/vfs"); -/// For the refusals of writes to a remote path, which tests trigger on -/// purpose; every other err here still fails a test (`scopedLog`). -const refusal_log = base_module.scopedLog(.@"zml/vfs"); -/// Indices 0-2 are the standard streams, so `std.Io.File.stdout()` and -/// friends resolve to their own descriptor; the cwd placeholder follows. -const CWD_HANDLE: u32 = 3; -/// Local files are the VFS's own: a bare path or a `file://` URI resolves -/// to the inner `Io`, never to a registered backend. -const local_scheme = "file"; +const CWD_HANDLE: u32 = 0; const VFS = @This(); -const Handle = struct { - handle: u32, - backend_idx: ?usize, - flags: std.Io.File.Flags = .{ .nonblocking = false }, - /// A local file's read mode (`useDirectIo`). - direct: direct_io.Mode = .{}, - - fn innerFile(self: *const Handle) std.Io.File { - return .{ .handle = @intCast(self.handle), .flags = self.flags }; - } -}; +const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, -backends: std.StringArrayHashMapUnmanaged(Backend) = .empty, +backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, handles: stdx.SegmentedList(Handle, 128) = .{}, closed_handles: std.ArrayList(u32) = .empty, @@ -91,11 +29,10 @@ pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); var handles: @FieldType(@This(), "handles") = .{}; + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); - std.debug.assert(handles.len == CWD_HANDLE + 1); return .{ .allocator = allocator, @@ -110,135 +47,49 @@ pub fn deinit(self: *VFS) void { self.backends.deinit(self.allocator); } -/// Serves `scheme` with `backend`, replacing any backend registered under -/// it. `scheme` is kept, not copied: it must outlive the VFS. -pub fn registerBackend(self: *VFS, scheme: []const u8, backend: Backend) std.mem.Allocator.Error!void { - std.debug.assert(!std.mem.eql(u8, scheme, local_scheme)); // file is the VFS's own (local_scheme) +pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } -/// The remote schemes on one HTTP client: `https`, `hf`, `s3` and `gs`, each -/// configured from the environment through its backend's `auto` (plain -/// `HTTP.init` for `https`), the client honoring the proxy variables. -/// Initialized in place, since the backends point at the client; it must not -/// move afterwards. Close every file opened through these schemes before -/// `deinit` (deinitialize a `Loader` first); the VFS must not reach them -/// after it. Their errors follow this module's contract. -pub const Remotes = struct { - /// The proxies' allocations, which must outlive the client. - arena: std.heap.ArenaAllocator, - client: std.http.Client, - https: HTTP, - hf: HF, - s3: S3, - gs: GCS, +pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - /// Builds the client on the VFS's base io and registers the four - /// schemes, replacing any backend registered under them. `environ` is - /// read here only: tokens, credentials and proxies are copied. On error - /// nothing is registered. - pub fn init(self: *Remotes, allocator: std.mem.Allocator, vfs: *VFS, environ: *std.process.Environ.Map) !void { - const base_io = vfs.base.inner; - self.arena = .init(allocator); - errdefer self.arena.deinit(); - self.client = .{ .allocator = allocator, .io = base_io }; - errdefer self.client.deinit(); - try self.client.initDefaultProxies(self.arena.allocator(), environ); - self.https = try .init(allocator, base_io, &self.client, .{ .protocol = .https }); - errdefer self.https.deinit(); - self.hf = try .auto(allocator, base_io, &self.client, environ); - errdefer self.hf.deinit(); - self.s3 = try .auto(allocator, base_io, &self.client, environ); - errdefer self.s3.deinit(); - self.gs = try .auto(allocator, base_io, &self.client, environ); - errdefer self.gs.deinit(); + return self.backends.remove(scheme); +} - vfs.mutex.lockUncancelable(base_io); - defer vfs.mutex.unlock(base_io); - // Room first: once a scheme is registered, nothing can fail. - try vfs.backends.ensureUnusedCapacity(vfs.allocator, 4); - vfs.backends.putAssumeCapacity("https", self.https.backend()); - vfs.backends.putAssumeCapacity("hf", self.hf.backend()); - vfs.backends.putAssumeCapacity("s3", self.s3.backend()); - vfs.backends.putAssumeCapacity("gs", self.gs.backend()); - } - - pub fn deinit(self: *Remotes) void { - self.gs.deinit(); - self.s3.deinit(); - self.hf.deinit(); - self.https.deinit(); - self.client.deinit(); - self.arena.deinit(); - } -}; - -/// The `std.Io` of this VFS. It points into the VFS, which must not move -/// once this was called. pub fn io(self: *VFS) std.Io { return .{ .userdata = &self.base, - .vtable = &io_vtable, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } -/// The VFS behind an `Io`, or null for any other implementation: a reader -/// that opened a file through `any_io` asks this VFS about that handle. -pub fn fromIo(any_io: std.Io) ?*VFS { - if (any_io.vtable != &io_vtable) return null; - return @fieldParentPtr("base", VFSBase.as(any_io.userdata)); -} - -const io_vtable: std.Io.VTable = VFSBase.vtable(.{ - .operate = operate, - .dirCreateDir = dirCreateDir, - .dirCreateDirPath = dirCreateDirPath, - .dirCreateDirPathOpen = dirCreateDirPathOpen, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirCreateFileAtomic = dirCreateFileAtomic, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .dirDeleteFile = dirDeleteFile, - .dirDeleteDir = dirDeleteDir, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, -}); - -/// The read hints of the backend serving `path`, for a reader sizing its -/// requests. A path resolves as it does for `openFile`: a bare path or a -/// `file://` URI is local (`ReadHints.local`), and a scheme no backend -/// serves is `VFSNotRegistered`. -pub fn readHints(self: *VFS, path: []const u8) !ReadHints { - const backend_idx, _, _ = try self.lookupDir(.cwd(), path); - const index = backend_idx orelse return .local; - return self.backends.entries.items(.value)[index].read_hints; -} - -/// Whether `path` resolves to a local file, the only kind `useDirectIo` can -/// read directly; an unregistered scheme is not. Opens nothing. -pub fn isLocal(self: *VFS, path: []const u8) bool { - const backend_idx, _, _ = self.lookupDir(.cwd(), path) catch return false; - return backend_idx == null; -} - fn openHandle(self: *VFS) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -266,53 +117,6 @@ fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { return .{ handle, self.getBackend(handle.backend_idx) }; } -/// Direct reads for a local file under `policy`: sets the flag on a -/// read-only descriptor of a regular file at least one alignment unit long -/// (and, under `auto`, mostly out of the page cache) and returns true, after -/// which every read of the file must meet `direct_io_alignment` in offset, -/// buffers and total. False leaves the file buffered: another backend's -/// file, `off`, a cached file under `auto`, a filesystem that refuses the -/// flag, or a file whose direct read the kernel already rejected. The first -/// call under `on` or `auto` decides for the handle's life -/// (`direct_io.Mode`); the caller must ask before it reads. The mechanism -/// lives in the VFS because it owns the descriptor; one descriptor suffices -/// because a reader reads a file in one mode. -pub fn useDirectIo(self: *VFS, file: std.Io.File, policy: DirectIo) bool { - const handle, _ = self.getFileHandle(file); - if (handle.backend_idx != null) return false; - return handle.direct.decide(self.base.inner, handle.innerFile(), policy); -} - -/// See `predictDirectIo`. -pub const DirectIoPrediction = direct_io.Prediction; - -/// What `useDirectIo` would answer under `policy` for the local file -/// `sub_path` of `dir`, and its size, asked on a descriptor of its own that -/// is closed again: no handle is decided and nothing is logged. For a reader -/// that places memory before it plans its files. Under `auto` this sample -/// caches what it misses, so a decision that follows may find any file -/// cached and read it buffered, the likelier the fewer pages it has per -/// sample (`direct_io.predict`). Null for a path that is not local, or a -/// file that does not open or stat. -pub fn predictDirectIo(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, policy: DirectIo) error{Canceled}!?DirectIoPrediction { - const backend_idx, const inner_dir, const inner = self.lookupDir(dir, sub_path) catch return null; - if (backend_idx != null) return null; - const file = inner_dir.openFile(inner, stripScheme(sub_path), .{ .mode = .read_only }) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => return null, - }; - defer file.close(inner); - return direct_io.predict(inner, file, policy); -} - -/// A local file as the inner `Io` may read it, off direct reads -/// (`direct_io.Mode.leave`). -fn innerReadable(self: *VFS, handle: *Handle) std.Io.File { - const file = handle.innerFile(); - handle.direct.leave(self.base.inner, file); - return file; -} - fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); @@ -326,7 +130,7 @@ fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { } fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx].io else return self.base.inner; + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; } fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { @@ -337,7 +141,6 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz if (sub_path) |sp| { if (std.mem.indexOf(u8, sp, "://") != null) { const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; - if (std.mem.eql(u8, uri.scheme, local_scheme)) return .{ null, std.Io.Dir.cwd(), self.base.inner }; const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { if (std.mem.eql(u8, uri.scheme, s)) break idx; } else return error.VFSNotRegistered; @@ -346,8 +149,19 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } if (std.meta.eql(dir, std.Io.Dir.cwd())) { - // Without "://", a path from cwd is local, a colon in it included. - return .{ null, dir, self.base.inner }; + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; + + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } } else { const handle = self.getDirHandle(dir); if (handle.backend_idx) |backend_idx| { @@ -358,26 +172,9 @@ fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usiz } } -/// `path` without its "scheme://" prefix; a name with a colon and no "://" -/// is a plain local name. fn stripScheme(path: []const u8) []const u8 { - const i = std.mem.indexOf(u8, path, "://") orelse return path; - return path[i + 3 ..]; -} - -/// The inner `Io`'s directory for creating or deleting `sub_path` from -/// `dir`. A remote path is refused: no remote backend writes, and the base -/// layer under each one would write a local file in its place. -fn writableDir(self: *VFS, dir: std.Io.Dir, sub_path: []const u8, operation: []const u8) error{ ReadOnlyFileSystem, Unexpected }!std.Io.Dir { - const backend_idx, const dir_, _ = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend to {s} '{s}' : {any}", .{ operation, sub_path, err }); - return error.Unexpected; - }; - if (self.getScheme(backend_idx)) |scheme| { - refusal_log.err("Cannot {s} '{s}': {s} paths are read-only", .{ operation, sub_path, scheme }); - return error.ReadOnlyFileSystem; - } - return dir_; + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; } fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { @@ -386,7 +183,7 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .file_read_streaming => |o| { const handle, const backend = self.getFileHandle(o.file); return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = self.innerReadable(handle), + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, .data = o.data, } }); }, @@ -405,32 +202,6 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable } } -fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "create directory"); - return self.base.inner.vtable.dirCreateDir(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); -} - -fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "create directory"); - return self.base.inner.vtable.dirCreateDirPath(self.base.inner.userdata, dir_, stripScheme(sub_path), permissions); -} - -fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "create directory"); - const inner = self.base.inner; - const fs_dir = try inner.vtable.dirCreateDirPathOpen(inner.userdata, dir_, stripScheme(sub_path), permissions, options); - errdefer fs_dir.close(inner); - const idx, const handle = self.openHandle() catch return std.Io.Dir.CreateDirPathOpenError.Unexpected; - handle.* = .{ - .handle = @intCast(fs_dir.handle), - .backend_idx = null, - }; - return .{ .handle = @intCast(idx) }; -} - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -475,36 +246,28 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "create file"); - const inner = self.base.inner; - const file = try inner.vtable.dirCreateFile(inner.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(inner); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + + const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), - .backend_idx = null, + .backend_idx = backend_idx, .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = file.flags }; } -/// The atomic file holds the inner `Io`'s handles, not this VFS's. -fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "create file"); - return self.base.inner.vtable.dirCreateFileAtomic(self.base.inner.userdata, dir_, stripScheme(sub_path), options); -} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; - if (flags.mode != .read_only and backend_idx != null) { - refusal_log.err("Cannot open '{s}' for writing: {s} paths are read-only", .{ sub_path, self.getScheme(backend_idx).? }); - return error.ReadOnlyFileSystem; - } const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; @@ -567,25 +330,13 @@ fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8 if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer[prefix.len..]); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); return prefix.len + path_len; } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, stripScheme(path_name), out_buffer); + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); } } -fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "delete file"); - return self.base.inner.vtable.dirDeleteFile(self.base.inner.userdata, dir_, stripScheme(sub_path)); -} - -fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const dir_ = try self.writableDir(dir, sub_path, "delete directory"); - return self.base.inner.vtable.dirDeleteDir(self.base.inner.userdata, dir_, stripScheme(sub_path)); -} - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); @@ -625,7 +376,7 @@ fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []co const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = self.innerReadable(src_handle); + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -646,7 +397,7 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c const original_src_io = reader.io; const original_src_handle = reader.file; reader.io = src_backend; - reader.file = self.innerReadable(src_handle); + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; defer { reader.io = original_src_io; reader.file = original_src_handle; @@ -658,7 +409,6 @@ fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []c fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - if (try handle.direct.read(self.base.inner, handle.innerFile(), data, offset)) |bytes| return bytes; return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); } @@ -687,281 +437,6 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. } } -test "VFS gives the read hints of local and registered paths" { - var filesystem = try VFS.init(std.testing.allocator, std.testing.io); - defer filesystem.deinit(); - const hints: ReadHints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true }; - try filesystem.registerBackend("test", .{ .io = std.testing.io, .read_hints = hints }); - - try std.testing.expectEqual(hints, try filesystem.readHints("test://bucket/object")); - try std.testing.expect(!filesystem.isLocal("test://bucket/object")); - - for ([_][]const u8{ "/tmp/model.safetensors", "models/model.safetensors", "file:///tmp/model.safetensors" }) |path| { - try std.testing.expectEqual(ReadHints{ .read_chunk_size = 8 * 1024 * 1024, .high_latency = false }, try filesystem.readHints(path)); - try std.testing.expect(filesystem.isLocal(path)); - } - try std.testing.expectError(error.VFSNotRegistered, filesystem.readHints("missing://bucket/object")); - try std.testing.expect(!filesystem.isLocal("missing://bucket/object")); - try std.testing.expectEqual(@as(?*VFS, &filesystem), VFS.fromIo(filesystem.io())); - try std.testing.expectEqual(@as(?*VFS, null), VFS.fromIo(std.testing.io)); -} - -test "Remotes registers every remote scheme with its read hints" { - var filesystem = try VFS.init(std.testing.allocator, std.testing.io); - defer filesystem.deinit(); - // No credentials and no proxies: nothing is read from the user's home. - var environ: std.process.Environ.Map = .init(std.testing.allocator); - defer environ.deinit(); - var remotes: Remotes = undefined; - try remotes.init(std.testing.allocator, &filesystem, &environ); - defer remotes.deinit(); - - const Case = struct { path: []const u8, local: bool = false, hints: ReadHints }; - const remote: ReadHints = .{ .read_chunk_size = 16 * 1024 * 1024, .high_latency = true }; - const cases = [_]Case{ - .{ .path = "file:///tmp/model", .local = true, .hints = .local }, - .{ .path = "/var/models/model", .local = true, .hints = .local }, - .{ .path = "https://example.com/model", .hints = remote }, - .{ .path = "s3://bucket/model", .hints = remote }, - .{ .path = "gs://bucket/model", .hints = remote }, - .{ .path = "hf://owner/model", .hints = .{ .read_chunk_size = 32 * 1024 * 1024, .high_latency = true } }, - }; - for (cases) |case| { - try std.testing.expectEqual(case.hints, try filesystem.readHints(case.path)); - // Only a local file can be read directly; nothing is opened to ask. - try std.testing.expectEqual(case.local, filesystem.isLocal(case.path)); - if (case.local) continue; - // A remote real path carries its scheme once, resolved without a request. - var real_path: [64]u8 = undefined; - try std.testing.expectEqualStrings(case.path, real_path[0..try std.Io.Dir.cwd().realPathFile(filesystem.io(), case.path, &real_path)]); - } -} - -test "VFS serves bare paths and file:// URIs from the inner Io" { - const testing_io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const created = try tmp.dir.createFile(testing_io, "bare.bin", .{ .read = true }); - try created.writePositionalAll(testing_io, "bare path bytes", 0); - var path_buffer: [std.fs.max_path_bytes]u8 = undefined; - const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; - created.close(testing_io); - - var filesystem = try VFS.init(std.testing.allocator, testing_io); - defer filesystem.deinit(); - const vfs_io = filesystem.io(); - - var uri_buffer: [std.fs.max_path_bytes + 8]u8 = undefined; - const uri = try std.fmt.bufPrint(&uri_buffer, "file://{s}", .{path}); - for ([_][]const u8{ path, uri }) |open_path| { - var real_buffer: [std.fs.max_path_bytes]u8 = undefined; - try std.testing.expectEqualStrings(path, real_buffer[0..try std.Io.Dir.cwd().realPathFile(vfs_io, open_path, &real_buffer)]); - const opened = try std.Io.Dir.openFile(.cwd(), vfs_io, open_path, .{ .mode = .read_only }); - defer opened.close(vfs_io); - const handle, _ = filesystem.getFileHandle(opened); - try std.testing.expectEqual(@as(?usize, null), handle.backend_idx); - // A file this small is never direct. - try std.testing.expect(!filesystem.useDirectIo(opened, .on)); - var contents: [15]u8 = undefined; - try std.testing.expectEqual(contents.len, try opened.readPositionalAll(vfs_io, &contents, 0)); - try std.testing.expectEqualStrings("bare path bytes", &contents); - } -} - -test "VFS reads the same bytes under every direct I/O policy" { - const testing_io = std.testing.io; - const alignment = 4096; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var contents: [3 * alignment + 100]u8 = undefined; - for (&contents, 0..) |*byte, i| byte.* = @truncate(i * 7 + 3); - const created = try tmp.dir.createFile(testing_io, "data.bin", .{ .read = true }); - try created.writePositionalAll(testing_io, &contents, 0); - var path_buffer: [std.fs.max_path_bytes]u8 = undefined; - const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; - created.close(testing_io); - - var filesystem = try VFS.init(std.testing.allocator, testing_io); - defer filesystem.deinit(); - const vfs_io = filesystem.io(); - const buffer = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(alignment), 2 * alignment); - defer std.testing.allocator.free(buffer); - const plain: []u8 = buffer; - for ([_]DirectIo{ .on, .auto, .off }) |policy| { - const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); - defer file.close(vfs_io); - // Direct when the policy and the filesystem allow it, buffered - // otherwise: `off`, or under `auto` a file just written and thus - // cached. The bytes are the same either way. - const direct = filesystem.useDirectIo(file, policy); - if (policy != .on) try std.testing.expect(!direct); - // The decision stands; `off` neither asks nor changes it. - try std.testing.expect(!filesystem.useDirectIo(file, .off)); - try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); - - // Aligned offset, buffer and length. - @memset(buffer, 0); - try std.testing.expectEqual(buffer.len, try file.readPositionalAll(vfs_io, buffer, alignment)); - try std.testing.expectEqualSlices(u8, contents[alignment..][0..buffer.len], buffer); - - // One aligned call that runs past the end of the file is cut there. - @memset(buffer, 0); - const past_end = try file.readPositional(vfs_io, &.{plain}, 2 * alignment); - try std.testing.expectEqual(contents.len - 2 * alignment, past_end); - try std.testing.expectEqualSlices(u8, contents[2 * alignment ..], buffer[0..past_end]); - try std.testing.expectEqual(direct, filesystem.useDirectIo(file, policy)); - - // A streaming read takes the file off direct reads for good. (A - // misaligned positional read of a direct file does too, with an - // error logged, which the test runner would count; a filesystem - // such as tmpfs may serve it instead, so it is not exercised here.) - var head: [8]u8 = undefined; - const head_slice: []u8 = &head; - try std.testing.expectEqual(head.len, try file.readStreaming(vfs_io, &.{head_slice})); - try std.testing.expectEqualSlices(u8, contents[0..head.len], &head); - try std.testing.expect(!filesystem.useDirectIo(file, policy)); - - // Buffered from now on: unaligned reads and the end of the file. - var tail: [50]u8 = undefined; - try std.testing.expectEqual(tail.len, try file.readPositionalAll(vfs_io, &tail, alignment + 3)); - try std.testing.expectEqualSlices(u8, contents[alignment + 3 ..][0..tail.len], &tail); - try std.testing.expectEqual(@as(usize, 0), try file.readPositional(vfs_io, &.{plain}, contents.len)); - } - - // A descriptor opened for writing is never direct: the flag would - // constrain its writes too. - const writable = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_write }); - defer writable.close(vfs_io); - try std.testing.expect(!filesystem.useDirectIo(writable, .on)); - const writable_handle, _ = filesystem.getFileHandle(writable); - if (comptime direct_io.supported) try std.testing.expectEqual(std.os.linux.E.BADF, direct_io.enable(writable_handle.innerFile().handle)); -} - -test "predictDirectIo answers as useDirectIo would and decides no handle" { - if (comptime !direct_io.supported) return error.SkipZigTest; - const testing_io = std.testing.io; - const alignment = 4096; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const contents: [3 * alignment + 100]u8 = @splat(0x5a); - try tmp.dir.writeFile(testing_io, .{ .sub_path = "data.bin", .data = &contents }); - try tmp.dir.writeFile(testing_io, .{ .sub_path = "short.bin", .data = contents[0..100] }); - var root_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = root_buffer[0..try tmp.dir.realPath(testing_io, &root_buffer)]; - - var filesystem = try VFS.init(std.testing.allocator, testing_io); - defer filesystem.deinit(); - // Another backend's scheme, served here by the plain io: its paths open, - // but they are not local. - try filesystem.registerBackend("remote", .{ .io = testing_io }); - const vfs_io = filesystem.io(); - const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); - defer dir.close(vfs_io); - var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buffer, "{s}/data.bin", .{root}); - - for ([_]DirectIo{ .on, .auto, .off }) |policy| { - // No handle is opened or decided to answer. - const handles = filesystem.handles.len; - const closed_handles = filesystem.closed_handles.items.len; - const predicted = (try filesystem.predictDirectIo(.cwd(), path, policy)).?; - try std.testing.expectEqual(@as(u64, contents.len), predicted.size); - try std.testing.expectEqual(predicted, (try filesystem.predictDirectIo(dir, "data.bin", policy)).?); - try std.testing.expectEqual(handles, filesystem.handles.len); - try std.testing.expectEqual(closed_handles, filesystem.closed_handles.items.len); - - // A fresh handle decides as predicted: direct only under `on`, where - // the filesystem allows it, since the file was just written and so - // is cached. - const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); - defer file.close(vfs_io); - try std.testing.expectEqual(predicted.direct, filesystem.useDirectIo(file, policy)); - if (policy != .on) try std.testing.expect(!predicted.direct); - } - - // A file shorter than a unit is never direct. - const short = (try filesystem.predictDirectIo(dir, "short.bin", .on)).?; - try std.testing.expectEqual(DirectIoPrediction{ .size = 100, .direct = false }, short); - // Nothing to answer for a remote path, though its backend opens the same - // file, or for a missing file. - var remote_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; - const remote = try std.fmt.bufPrint(&remote_buffer, "remote://{s}/data.bin", .{root}); - try std.testing.expectEqual(null, try filesystem.predictDirectIo(.cwd(), remote, .on)); - try std.testing.expectEqual(null, try filesystem.predictDirectIo(dir, "missing.bin", .on)); -} - -test "auto keeps reading a cold file direct after probing it" { - if (comptime !direct_io.supported) return error.SkipZigTest; - const testing_io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - // One 2 MiB window per sample (`direct_io.residency_samples`): many - // pages each, on 64 KiB-page kernels too. - const window = 2 << 20; - const contents = try std.testing.allocator.alloc(u8, direct_io.residency_samples * window); - defer std.testing.allocator.free(contents); - @memset(contents, 0x5a); - const created = try tmp.dir.createFile(testing_io, "cold.bin", .{ .read = true }); - try created.writePositionalAll(testing_io, contents, 0); - try created.sync(testing_io); - _ = std.os.linux.fadvise(created.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); - var path_buffer: [std.fs.max_path_bytes]u8 = undefined; - const path = path_buffer[0..try created.realPath(testing_io, &path_buffer)]; - created.close(testing_io); - - // The testing io, but the probes draw shifts a quarter window apart, - // 512 KiB from each other's samples. At random shifts a probe may sample - // the pages an earlier one's misses read, and find the file cached - // (`direct_io.predict`): about once in window / page pairs of probes. - const Shifts = struct { - var drawn: u64 = 0; - - fn random(_: ?*anyopaque, buffer: []u8) void { - std.mem.writeInt(u64, buffer[0..8], window / 8 + drawn * window / 4, .native); - drawn += 1; - } - }; - Shifts.drawn = 0; - var vtable = testing_io.vtable.*; - vtable.random = Shifts.random; - const shifting_io: std.Io = .{ .userdata = testing_io.userdata, .vtable = &vtable }; - - var filesystem = try VFS.init(std.testing.allocator, shifting_io); - defer filesystem.deinit(); - const vfs_io = filesystem.io(); - for (0..2) |open| { - // Predicted first, as a reader placing memory before it plans. - const predicted = (try filesystem.predictDirectIo(.cwd(), path, .auto)).?; - const file = try std.Io.Dir.openFile(.cwd(), vfs_io, path, .{ .mode = .read_only }); - defer file.close(vfs_io); - const direct = filesystem.useDirectIo(file, .auto); - // A filesystem that kept the file cached (tmpfs) or refused direct - // reads says nothing here. - if (open == 0 and !direct) return error.SkipZigTest; - try std.testing.expect(predicted.direct); - // The first probes' misses queued reads of the pages they sampled; - // once they landed, sampling the same pages would find the file - // cached. - if (open == 0) try testing_io.sleep(.fromMilliseconds(250), .awake); - try std.testing.expect(direct); - } - // Each probe drew a shift of its own. - try std.testing.expectEqual(@as(u64, 4), Shifts.drawn); -} - -test "standard streams resolve to their own descriptors" { - var vfs: VFS = try .init(std.testing.allocator, std.testing.io); - defer vfs.deinit(); - - const files = [_]std.Io.File{ .stdin(), .stdout(), .stderr() }; - const fds = [_]std.posix.fd_t{ std.posix.STDIN_FILENO, std.posix.STDOUT_FILENO, std.posix.STDERR_FILENO }; - for (files, fds) |file, fd| { - const handle, const backend = vfs.getFileHandle(file); - try std.testing.expectEqual(fd, handle.innerFile().handle); - try std.testing.expect(backend.vtable == vfs.base.inner.vtable); - } -} - test "copy a local file through the VFS" { const allocator = std.testing.allocator; const process_io = std.testing.io; @@ -982,9 +457,11 @@ test "copy a local file through the VFS" { const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); defer allocator.free(destination_uri); - // `file://` is the VFS's own scheme: no backend to register. + var file_backend: File = .init(allocator, process_io, .{}); + defer file_backend.deinit(); var vfs: VFS = try .init(allocator, process_io); defer vfs.deinit(); + try vfs.register("file", file_backend.io()); const vfs_io = vfs.io(); { @@ -1004,107 +481,3 @@ test "copy a local file through the VFS" { defer allocator.free(copied); try std.testing.expectEqualSlices(u8, &source_data, copied); } - -test "creates and deletes on a local path go to the inner Io, from a path or an open directory" { - const testing_io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var path_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; - var vfs: VFS = try .init(std.testing.allocator, testing_io); - defer vfs.deinit(); - const vfs_io = vfs.io(); - - var uri_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; - { - const opened = try std.Io.Dir.cwd().createDirPathOpen(vfs_io, try std.fmt.bufPrint(&uri_buffer, "file://{s}/a/b", .{root}), .{}); - defer opened.close(vfs_io); - // Relative to the descriptor the VFS's handle stands for. - try opened.createDir(vfs_io, "c", .default_dir); - try opened.writeFile(vfs_io, .{ .sub_path = "c/file", .data = "bytes" }); - try tmp.dir.access(testing_io, "a/b/c/file", .{}); - try opened.deleteFile(vfs_io, "c/file"); - try opened.deleteDir(vfs_io, "c"); - // A colon without "://" is part of a local name. - try opened.writeFile(vfs_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); - try tmp.dir.access(testing_io, "a/b/notes:old.txt", .{}); - try opened.deleteFile(vfs_io, "notes:old.txt"); - try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/notes:old.txt", .{})); - } - try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/b/c", .{})); - try std.Io.Dir.cwd().createDirPath(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); - try tmp.dir.access(testing_io, "a/d", .{}); - try std.Io.Dir.cwd().deleteDir(vfs_io, try std.fmt.bufPrint(&uri_buffer, "{s}/a/d", .{root})); - try std.testing.expectError(error.FileNotFound, tmp.dir.access(testing_io, "a/d", .{})); -} - -test "creates and deletes on a remote path fail and write nothing anywhere" { - const allocator = std.testing.allocator; - const testing_io = std.testing.io; - var object: [16]u8 = @splat(0x2a); - var server = try MockServer.init(testing_io, &object, .{}); - defer server.deinit(testing_io); - try server.start(testing_io); - - var client: std.http.Client = .{ .allocator = allocator, .io = testing_io }; - defer client.deinit(); - var endpoint_buffer: [64]u8 = undefined; - const endpoint = try std.fmt.bufPrint(&endpoint_buffer, "http://127.0.0.1:{d}", .{server.port()}); - var s3 = try S3.init(allocator, testing_io, &client, .{ .endpoint_url = endpoint, .region = "us-east-1" }); - defer s3.deinit(); - var vfs: VFS = try .init(allocator, testing_io); - defer vfs.deinit(); - try vfs.registerBackend("s3", s3.backend()); - const vfs_io = vfs.io(); - const cwd = std.Io.Dir.cwd(); - - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFile(vfs_io, "s3://bucket/key", .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.openFile(vfs_io, "s3://bucket/key", .{ .mode = .read_write })); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createFileAtomic(vfs_io, "s3://bucket/key", .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDir(vfs_io, "s3://bucket/dir", .default_dir)); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPath(vfs_io, "s3://bucket/a/b")); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.createDirPathOpen(vfs_io, "s3://bucket/a/b", .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteFile(vfs_io, "s3://bucket/key")); - try std.testing.expectError(error.ReadOnlyFileSystem, cwd.deleteDir(vfs_io, "s3://bucket/dir")); - // A directory opened on the remote is no way around it. - const bucket = try cwd.openDir(vfs_io, "s3://bucket", .{}); - defer bucket.close(vfs_io); - try std.testing.expectError(error.ReadOnlyFileSystem, bucket.createFile(vfs_io, "key", .{})); - try std.testing.expectError(error.ReadOnlyFileSystem, bucket.deleteFile(vfs_io, "key")); - - // Nothing on local disk, where the base layer would have written, and - // no request to the server. - for ([_][]const u8{ "s3:", "bucket" }) |name| { - try std.testing.expectError(error.FileNotFound, cwd.access(testing_io, name, .{})); - } - try server.stop(testing_io); - try std.testing.expectEqual(0, server.head_requests.load(.acquire) + server.get_requests.load(.acquire)); -} - -test "a name with a colon and no \"://\" is local" { - const testing_io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(testing_io, .{ .sub_path = "notes:old.txt", .data = "bytes" }); - var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const root = path_buffer[0..try tmp.dir.realPath(testing_io, &path_buffer)]; - - var vfs: VFS = try .init(std.testing.allocator, testing_io); - defer vfs.deinit(); - const vfs_io = vfs.io(); - - const backend_idx, _, _ = try vfs.lookupDir(.cwd(), "notes:old.txt"); - try std.testing.expectEqual(null, backend_idx); - - const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); - defer dir.close(vfs_io); - const file = try dir.openFile(vfs_io, "notes:old.txt", .{}); - defer file.close(vfs_io); - var contents: [5]u8 = undefined; - try std.testing.expectEqual(contents.len, try file.readPositionalAll(vfs_io, &contents, 0)); - try std.testing.expectEqualStrings("bytes", &contents); - - var real_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const real_path = real_path_buffer[0..try dir.realPathFile(vfs_io, "notes:old.txt", &real_path_buffer)]; - try std.testing.expect(std.mem.endsWith(u8, real_path, "/notes:old.txt")); -} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index bac383e7..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,6 +78,7 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", + "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -86,20 +87,13 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "io/ShardGrid.zig", - "io/Source.zig", - "io/TensorStore.zig", - "io/backend.zig", - "io/buffered_loader.zig", - "io/direct_loader.zig", - "io/dma_calibration.zig", - "io/host_memory.zig", - "io/loader.zig", - "io/log.zig", - "kernel.zig", "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", + "kernels/moe/weights.zig", + "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", @@ -119,11 +113,12 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/cutlass_flashinfer", + "//zml/kernels/moe/flashinfer_cutlass", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", @@ -195,11 +190,6 @@ zig_library( "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": [ - "//platforms/rocm", - ], - "//conditions:default": [], }), ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d1455789..93eafa8d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -23,6 +23,7 @@ const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; +const TensorStore = @import("io.zig").TensorStore; const Compiler = @This(); const log = std.log.scoped(.@"zml/Compiler"); @@ -31,6 +32,7 @@ allocator: std.mem.Allocator, io: std.Io, arena: std.heap.ArenaAllocator, +options: Options, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, @@ -77,6 +79,7 @@ pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, meshes: []const *const Sharding.Mesh = &.{}, + store: ?*const TensorStore = null, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, /// HACK: Extra bytes added to the device memory size reported to XLA. @@ -84,8 +87,7 @@ pub const Options = struct { /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. /// XLA then counts them against device memory, leaves no budget for temporary buffers and /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate: `zml.io.TensorStore.View.hostMappedBytes` for those the loader delivers - /// `.host_mapped`. Remove once XLA properly supports host-pinned inputs. + /// compensate. Remove once XLA properly supports host-pinned inputs. extra_device_memory_bytes: u64 = 0, // Debugging options program_name: []const u8 = "zml", @@ -101,7 +103,7 @@ pub const Error = std.mem.Allocator.Error || mlir.Error || upb.SerializeError || pjrtx.Client.CompileError || - error{MissingDeviceInTile}; + error{ MissingDeviceInTile, OpenShardingAtBoundaries }; pub const Donation = union(enum) { implicit: u32, @@ -195,6 +197,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, + .options = opts, .partitioner = opts.partitioner orelse .fromTarget(platform.target), .meshes = meshes, .location = unknown_location, @@ -289,6 +292,7 @@ test pushLocation { .mlir_pass_manager = undefined, .mlir_known_types = undefined, .module = undefined, + .options = .{}, .platform = undefined, .partitioner = undefined, .meshes = undefined, @@ -362,16 +366,13 @@ pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mes /// Find a mesh that covers all the given axes pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { if (@TypeOf(logical_axes) != []const Shape.Tag) { - const comp_tags = comptime Shape.parseTags(logical_axes); - const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); - return compiler.resolveMesh(parsed_tags); + return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); } var ok_meshe: ?*const Sharding.Mesh = null; for (compiler.meshes) |shd| { var covers_all: bool = true; - for (logical_axes) |ax| { - const input_axis = Shape.toTag(ax); + for (logical_axes) |input_axis| { var covers_this: bool = false; for (shd.logical.axes.slice()) |existing_axis| { if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { @@ -424,10 +425,7 @@ pub fn compile( ) Error!Exe { return switch (platform.io_impl) { .threaded => compileInternal(allocator, io, platform, func_, args, opts), - // A compile holds its thread for its whole run, so it takes a pool - // worker of its own instead of queuing behind other jobs; concurrent - // compiles run in parallel. - .zio => try zio.blockInPlaceReserved(struct { + .zio => try zio.blockInPlace(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); } @@ -452,7 +450,7 @@ pub fn compileInternal( // in mind we might want to revisit that later. // // This is NOT compatible with std.Io.Evented. - // It is compatible with zio, hence the blockInPlaceReserved above. + // It is compatible with zio, hence the blockInPlace above. const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, @@ -466,10 +464,7 @@ pub fn compileInternal( var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = compiler.emitMlir(func_, args) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => unreachable, - }; + var result = try compiler.emitMlir(func_, args); defer result.output_info.deinit(compiler.allocator); defer result.input_info.deinit(compiler.allocator); @@ -597,20 +592,21 @@ fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTup var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var result = blk: { + var output_info = blk: { compiler.activate(); defer compiler.deactivate(); - break :blk @call(.auto, func_, args); - }; - var output_info = try collectOutputInfo(compiler, fn_scope, &result); + var result = @call(.auto, func_, args); + // Output collection can emit replication constraints, so it is part of tracing. + break :blk try collectOutputInfo(compiler, fn_scope, &result); + }; errdefer output_info.deinit(compiler.allocator); // Finalize in a separate function that doesn't depend on `func` type. return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); } -fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CreateBlockArgumentsCb = struct { compiler: *Compiler, scope: *Scope, @@ -618,12 +614,23 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou current_argument_id: u32 = 0, infos: std.MultiArrayList(TensorInfo) = .empty, - fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { + fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { // Declare the program argument as a packed u8 const og_shape = tensor._shape; + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); + return error.OpenShardingAtBoundaries; + } const packed_shape = og_shape.packedShape(); - const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); - var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); + const mlir_ctx = ctx.compiler.mlir_ctx; + const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); + var location: *const mlir.Location = ctx.compiler.unknown_location; + if (ctx.compiler.options.store) |store| { + if (store.id_to_sources.get(tensor.id)) |sources| { + location = location.named(mlir_ctx, sources.tensors[0].name); + } + } + var value = ctx.scope.block.addArgument(mlir_type, location); // But immediately create the MLIR value corresponding to the fp4 data. // That way the user code only sees a proper fp4 tensor. @@ -654,7 +661,7 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou return context.infos; } -fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CollectOutputInfoCb = struct { compiler: *Compiler, scope: *Scope, @@ -662,9 +669,16 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) !void { - const og_shape = tensor.shape(); - var value = ctx.scope.id_to_argument.get(tensor.id) orelse - tensor._value orelse + // Outputs with no declared sharding default to replication. Constrain the + // returned value so the partitioner produces a replicated output buffer. + const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); + const og_shape = output.shape(); + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); + return error.OpenShardingAtBoundaries; + } + var value = ctx.scope.id_to_argument.get(output.id) orelse + output._value orelse @panic("no value found for output tensor"); if (tensor.dtype().bitSizeOf() < 8) { @@ -806,6 +820,7 @@ fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { // even if later it means that some mesh axis is never used. // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas // TODO(#914): decide if we should forbid this altogether + // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); } log.debug("Using {f}", .{m}); @@ -1019,6 +1034,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load platform.pjrt_client, platform.pjrt_api, compiler.allocator, + compiler.io, compiler.module, try upb.serialize(options, upb_arena), ); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index bc5e6afe..040216bc 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -21,10 +21,22 @@ mesh: ?*const Mesh, partition: Partitioning, pub const MAX_MESH_RANK = 4; + +/// Full replication across participating devices. +/// For buffers, each device holds a complete copy. Applied as a graph constraint, +/// this closes every dimension to partitioning. `Shape.init` uses this sharding. +/// Unconstrained graph results use `open` instead. pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; +/// Unspecified sharding for graph results. During compilation, the partitioner +/// may infer sharding from surrounding constraints or leave the value replicated. +/// Executable inputs require closed sharding. An executable output with this +/// unspecified sharding defaults to replication. An explicitly declared output +/// sharding containing open dimensions is rejected. +pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; + pub fn get(sharding: Sharding, ax: usize) PartitionSpec { - return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); + return sharding.partition.get(ax); } pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { @@ -52,7 +64,6 @@ pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { } pub fn isFullyReplicated(self: Sharding) bool { - if (self.mesh == null) return true; for (self.partition.toArray()) |spec| { if (spec != .replicated and spec != .out_of_bound) return false; } @@ -98,8 +109,18 @@ test isCompatible { } } +/// Formats the declared sharding, not the partitioner's inferred sharding. +/// Without a mesh, prints `{replicated}` for full replication or `{open}` for +/// unspecified graph sharding. +/// +/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: +/// - `data`: split this dimension over the logical mesh axis named `data`. +/// - `replicated`: keep this dimension whole on each shard. +/// - `?`: leave this dimension to partitioner inference during compilation. +/// - `@dp`: the mesh name. +/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); const specs = self.partition.toArray(); var rank = specs.len; while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; @@ -117,8 +138,10 @@ pub fn format(self: Sharding, writer: *std.Io.Writer) !void { try writer.print("}}@{s}", .{mesh.name}); } -/// Describes how a given Shape axis behaves inside a mesh. -/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +/// Describes how one tensor dimension is distributed across a logical mesh. +/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. +/// In a graph constraint, `.open` lets the partitioner choose either behavior. +/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. pub const PartitionSpec = enum(u4) { mesh_axis_0 = 0, mesh_axis_1 = 1, @@ -405,122 +428,110 @@ pub const Partitioning = packed struct { try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); } - /// A reusable, mesh-independent partitioning keyed by tensor axis tags. - /// Values name logical mesh axes (eg `.model`), or are `.replicated` / `.open`. - /// It is resolved into a positional `Partitioning` once the tensor tags and the mesh are known, - /// so the same value can be applied to tensors with reordered axes, or on different meshes. - pub const Logical = struct { - pub const Spec = union(enum) { - /// Sharded along the logical mesh axis with this name. - axis: Shape.Tag, - replicated, - open, + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + const logical = if (@TypeOf(partitioning) == Logical) + partitioning + else + // capture all the comptime logic in Logical. + comptime Logical.parse(partitioning); - pub fn init(value: anytype) Spec { - const V = @TypeOf(value); - if (V == Spec) return value; - if (V == @EnumLiteral()) return switch (value) { - .replicated => .replicated, - .open => .open, - else => .{ .axis = Shape.toTag(value) }, - }; - if (V == Shape.Tag) return .{ .axis = value }; - stdx.debug.compileError("Partitioning.Logical.Spec expects a logical axis tag, .replicated or .open, got {}", .{V}); + return logical.applyUseAll(tags, mesh, default_value); + } + + pub const Logical = union(enum) { + /// Replicates every dimension, including when used as a graph constraint. + replicated, + mapping: struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + }, + + const replicated_tag: Shape.Tag = Shape.toTag(.replicated); + const open_tag: Shape.Tag = Shape.toTag(.open); + + pub fn parse(v: anytype) Logical { + const V = @TypeOf(v); + + var src_: Shape.TagsArray = .empty; + var tgt_: Shape.TagsArray = .empty; + + if (comptime stdx.meta.isStruct(V)) { + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); + tgt_.appendAssumeCapacity(@tagName(fv)); + + if (!comptime stdx.meta.isTuple(V)) { + src_.appendAssumeCapacity(field_name); + } + } + return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; } - pub fn resolve(self: Spec, mesh: *const Mesh) PartitionSpec { - return switch (self) { - .replicated => .replicated, - .open => .open, - .axis => |t| .sharded(@intCast(mesh.resolveLogicalAxis(t) orelse { - stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, t }); - })), - }; + stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); + } + + test "Logical.parse" { + const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); + + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); + } + + test "Logical.replicated" { + const mesh: Mesh = .initTest(.logical, .{.model}); + const tags: []const Shape.Tag = &.{ "a", "b" }; + const logical: Logical = .replicated; + for ([_]PartitionSpec{ .open, .replicated }) |default| { + try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); } - }; + const unspecified = Logical.parse(.{}); + try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); + } - const Entry = struct { tag: Shape.Tag, spec: Spec }; - - entries: stdx.BoundedArray(Entry, MAX_RANK) = .empty, - /// Spec of the axes without an entry. `null` defers to the default of the caller. - default: ?Spec = null, - - pub const replicated: Logical = .{ .default = .replicated }; - - /// Accepts a `Logical`, `.replicated`, or a struct mapping tensor axis tags to specs, - /// eg `.{ .out = .model, .in = .replicated }`. Null optional specs are skipped. - pub fn init(specs: anytype) Logical { - const T = @TypeOf(specs); - if (T == Logical) return specs; - if (T == @EnumLiteral()) return switch (specs) { - .replicated => Logical.replicated, - else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), + pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + const mapping = switch (logical.*) { + .replicated => return .{ .replicated(tags.len), null }, + .mapping => |*mapping_| mapping_, }; - stdx.debug.assertComptime(stdx.meta.isStruct(T), "Partitioning.Logical expects a struct of partition specs or .replicated, got {}", .{T}); - var result: Logical = .{}; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const value = @field(specs, field_name); - const spec: ?Spec = if (@typeInfo(@TypeOf(value)) == .optional) (if (value) |v| Spec.init(v) else null) else Spec.init(value); - if (spec) |s| result.entries.append(.{ .tag = Shape.toTag(field_name), .spec = s }) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); - } - return result; - } + var partition_: Partitioning = .repeat(default_value, tags.len); + var missing_src_axis: ?u8 = null; - /// Returns the spec of the given tensor axis, or `default` when it has no entry. - pub fn get(self: Logical, tag_: anytype) ?Spec { - const axis_tag = Shape.toTag(tag_); - for (self.entries.constSlice()) |entry| { - if (std.mem.eql(u8, std.mem.span(entry.tag), std.mem.span(axis_tag))) return entry.spec; - } - return self.default; - } - - /// Resolves the partitioning of a tensor with the given axis tags on `mesh`. - /// Every entry must name one of `tags`. - pub fn resolve(self: Logical, tags: []const Shape.Tag, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { - var res: Partitioning = .repeat(if (self.default) |d| d.resolve(mesh) else default_value, tags.len); - for (self.entries.constSlice()) |entry| { - const ax = Shape.axisFromTagMaybe(tags, entry.tag) orelse { - stdx.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), entry.tag }); + for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + missing_src_axis = @intCast(i); + continue; }; - res = res.set(ax, entry.spec.resolve(mesh)); + + const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) + .replicated + else if (mesh_tag == Logical.open_tag) + .open + else spec: { + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }; + partition_ = partition_.set(shape_ax, spec); } - return res; + return .{ partition_, missing_src_axis }; } - test "Logical resolves by tensor axis name on any mesh" { - const tp: Mesh = .initTest(.tp, .{.model}); - const ep: Mesh = .initTest(.ep, .{ .experts, .model }); - const specs: []const Spec = &.{ .init(.model), .replicated, .open }; - for (specs) |spec| { - const logical: Logical = .init(.{ .out = spec, .in = .replicated }); - const out = logical.get(.out).?; - try std.testing.expectEqual(std.meta.activeTag(spec), std.meta.activeTag(out)); - try std.testing.expectEqual(spec.resolve(&ep), out.resolve(&ep)); - try std.testing.expectEqual(null, logical.get(.batch)); + pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); - const tags: []const Shape.Tag = &.{ "in", "batch", "out" }; - try std.testing.expectEqualSlices( - PartitionSpec, - &.{ .replicated, .open, spec.resolve(&tp) }, - logical.resolve(tags, &tp, .open).toArray()[0..3], - ); - try std.testing.expectEqual(spec.resolve(&ep), logical.resolve(tags, &ep, .replicated).get(2)); + if (missing_shape_axis) |ax| { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); } - try std.testing.expectEqual(PartitionSpec.sharded(1), Spec.init(.model).resolve(&ep)); - } - - test "Logical replicated default and optional specs" { - const tp: Mesh = .initTest(.tp, .{.model}); - const logical: Logical = .init(.replicated); - try std.testing.expect(logical.get(.out).? == .replicated); - try std.testing.expectEqual(Partitioning.replicated(2), logical.resolve(&.{ "a", "b" }, &tp, .open)); - - const maybe: ?Spec = null; - const partial: Logical = .init(.{ .a = maybe, .b = @as(?Spec, .init(.model)) }); - try std.testing.expectEqual(1, partial.entries.len); - try std.testing.expectEqual(Partitioning.init(&.{ .open, .mesh_axis_0 }), partial.resolve(&.{ "a", "b" }, &tp, .open)); + return p; } }; }; @@ -1467,8 +1478,8 @@ pub const LogicalMesh = struct { /// * dimensions: dimension for each axis, -1 infers the size from underlying topology pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { if (@TypeOf(names) != []const Shape.Tag) { - const parsed_names = comptime Shape.parseTags(names); - return .init(@as([]const Shape.Tag, parsed_names.slice()), intents, dimensions); + const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); + return .init(parsed_names, intents, dimensions); } if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); @@ -1504,6 +1515,10 @@ pub const LogicalMesh = struct { return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; } + pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { + return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); + } + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { const target = std.mem.span(tag); for (self.axes.slice(), self.intents.slice()) |t, i| { @@ -1630,7 +1645,7 @@ pub const Mesh = struct { return .{ .name = @tagName(name_), .logical = .{ - .axes = .init(axes_tags.slice()), + .axes = .init(axes_tags), .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, .dimensions = @splat(-1), }, @@ -1669,13 +1684,17 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } + pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { + return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); + } + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (self.logical.axes.slice(), 0..) |axis, i| { + for (self.logical.axes.slice(), 0..) |ax, i| { const size = self.grid.axis_sizes[i]; tile_count *= size; - axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(axis), .size = @intCast(size) })); + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); } const replicas = @divExact(self.grid.device_count, tile_count); if (replicas > 1) { @@ -1686,8 +1705,8 @@ pub const Mesh = struct { for (0..self.grid.device_count) |partition_index| { const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; - for (0..self.logical.axes.len) |axis| { - tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; + for (0..self.logical.axes.len) |ax| { + tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; } // Device IDs here are executable partition indices, not PJRT device IDs. ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); @@ -1700,14 +1719,58 @@ pub const Mesh = struct { // There are at most MAX_MESH_RANK logical names, so one candidate is free. inline for (0..MAX_MESH_RANK + 1) |i| { const name = std.fmt.comptimePrint("__replica_{d}", .{i}); - const taken = for (self.logical.axes.slice()) |axis| { - if (std.mem.eql(u8, std.mem.span(axis), name)) break true; + const taken = for (self.logical.axes.slice()) |ax| { + if (std.mem.eql(u8, std.mem.span(ax), name)) break true; } else false; if (!taken) return name; } unreachable; } + /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute + /// referencing this mesh. Entries in `[...]` follow tensor dimension order. + /// Mesh axis indices in the specs resolve to this mesh's logical axis names. + /// The tensor rank and logical mesh rank are independent. + /// + /// Each dimension spec is encoded as follows: + /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over + /// logical mesh axis `i`. The dimension is closed to further partitioning. + /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is + /// closed, so the partitioner cannot assign a mesh axis to it. + /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. + /// It may acquire available mesh axes or remain replicated. + /// Closed therefore means no additional mesh axes, not necessarily replicated. + /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. + /// + /// Dimension replication (`{}`) applies to one tensor dimension. The optional + /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis + /// must carry identical tensor data across its coordinates and cannot be used + /// to split any dimension, including an open one. + /// + /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, + /// and a shape whose dimensions are ordered `batch`, `hidden`: + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .open } + /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> + /// ``` + /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. + /// The unused `model` axis stays available because an open dimension exists. + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> + /// ``` + /// Both dimensions are closed. Unused logical axes are explicitly replicated, + /// so devices differing only in their `model` coordinate hold the same shard. + /// + /// ``` + /// Partitioning: .{ .batch = .open, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{?}, {}]> + /// ``` + /// Only `batch` may acquire axes, and both `data` and `model` are available. + /// Adding `replicated={"data", "model"}` would prevent either axis from being + /// inferred for `batch`, despite its `{?}` entry. pub fn sdyShardingAttrForShape( data: *const Mesh, parent_allocator: std.mem.Allocator, @@ -1719,24 +1782,27 @@ pub const Mesh = struct { const allocator = arena.allocator(); const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); - for (0.., dimensions) |ax, *d| { - d.* = switch (shape.partition(ax)) { + const has_open_dimension = shape.containsPartitionSpec(.open); + for (0.., dimensions) |i, *d| { + d.* = switch (shape.partition(i)) { .out_of_bound => @panic("undefined"), .replicated => .replicated(ctx), .open => .open(ctx, &.{}), else => d: { - const axis = shape.partition(ax).meshAxis().?; - used[axis] = true; - break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(axis)))}); + const ax = shape.partition(i).meshAxis().?; + used[ax] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); }, }; } var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (data.logical.axes.slice(), 0..) |axis, i| { + for (data.logical.axes.slice(), 0..) |ax, i| { tile_count *= data.grid.axis_sizes[i]; - if (!used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + // Open dimensions can acquire unused logical axes during inference. + // Only reserve those axes for replication when every dimension is closed. + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -1752,8 +1818,8 @@ pub const Mesh = struct { var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; var tile_count: usize = 1; - for (0..shape.rank()) |ax| { - const size = if (shape.partition(ax).meshAxis()) |axis| self.grid.axis_sizes[axis] else 1; + for (0..shape.rank()) |i| { + const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; tile_shape.appendAssumeCapacity(size); tile_count *= size; } @@ -2030,7 +2096,7 @@ pub const DeviceGrid = struct { for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { if (used.contains(source)) continue; used.insert(source); - const info = mesh_.physical.axisInfo(source) orelse continue; + const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); } } @@ -2773,7 +2839,7 @@ test "mesh: open and replicated dimension mix" { .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"batch\", \"model\"}>", + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 49fe947b..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,6 +93,29 @@ pub const Metadata = union(BackendType) { } }; +pub const Options = @import("Options.zig"); + +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + q: Logical, + k: Logical, + v: Logical, + token_index: Logical = .replicated, + /// Omit to replicate backend metadata tensors. + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, + }, + out_specs: Logical, + + /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. + /// Other dimensions, token indices and backend metadata are replicated. + pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { + const heads: Logical = .parse(.{ .h = mesh_axis }); + return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; + } +}; + /// Causal attention as used in transformers. /// /// **Shapes**: @@ -104,19 +127,46 @@ pub const Metadata = union(BackendType) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return switch (parameters) { - .stablehlo => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); - - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - break :b attn_output; - }, - .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention), - }; +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); } + +/// `attention`, causal or not depending on `opts`. +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ + .in_specs = .{ + .q = partitioning.in_specs.q, + .k = partitioning.in_specs.k, + .v = partitioning.in_specs.v, + .token_index = partitioning.in_specs.token_index, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + }, + .out_specs = partitioning.out_specs, + }); +} + +const Attention = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + metadata: Metadata, + parameters: Parameters, + opts: Options, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .stablehlo => b: { + const seq_len = args.k.dim(.k); + const mask: ?zml.Tensor = if (args.opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) + else + null; + break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); + }, + .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), + }; + } +}; diff --git a/zml/buffer.zig b/zml/buffer.zig index 64d520a7..8714d80f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -27,8 +27,8 @@ test { /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. -/// * filled from a checkpoint by `zml.io.Loader` (docs/learn/loader.md). -/// * created from host data by `fromSlice` or `fromBytes`. +/// * loading weights from disk directly to the `device zml.aio.loadBuffers` +/// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { _platform: *const Platform, _shape: Shape, @@ -56,16 +56,6 @@ pub const Buffer = struct { pub fn devicePtr(self: *const Shard) *anyopaque { return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } - - /// Copies this shard's bytes to `destination`, which must hold the - /// shard's byte size, and waits for the copy. - pub fn toHost(self: *const Shard, io: std.Io, destination: []u8) !void { - const maybe_event = try self._pjrt_buffer.toHostBuffer(self._platform.pjrt_api, destination); - if (maybe_event) |event| { - defer event.deinit(self._platform.pjrt_api); - try event.await(self._platform.pjrt_api, io); - } - } }; pub const ShardIterator = struct { @@ -109,6 +99,17 @@ pub const Buffer = struct { }.deinit, {}, buffers); } + /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. + /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, + /// This only impacts device memory. + pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { + meta.visit(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); + } + /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; @@ -118,8 +119,6 @@ pub const Buffer = struct { return @intCast(self._shards.len); } - /// One shard per device of the shape's mesh (the platform's replicated - /// mesh when it names none), in that mesh's canonical device order. pub fn shards(self: *const Buffer) ShardIterator { return .{ ._platform = self._platform, @@ -157,6 +156,7 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const byte_size: usize = if (packed_shape.rank() == 0) packed_shape.dtype().sizeOf() else @@ -164,8 +164,8 @@ pub const Buffer = struct { stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); const base_ptr = data_.ptr; - for (placement.mesh().?.devicesInCanonicalOrder(), 0..) |device, device_index| { - const memory = deviceMemory(platform, device, opts.memory); + for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { + const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device .data = placement.shardPtr(device_index, base_ptr), @@ -248,8 +248,10 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - for (metadata.placement.mesh().?.devicesInCanonicalOrder()) |device| { - const memory = deviceMemory(platform, device, opts.memory); + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + + for (platform.physical_mesh.devices_in_canonical_order) |device| { + const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.CreateUninitializedBufferArgs = .{ // Change for each device .dst = .{ .memory = memory.pjrt_memory }, @@ -293,14 +295,14 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); const placement = placementOrPanic(self._platform, self._shape); - for (self._shards.constSlice(), 0..) |shard, shard_index| { + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(shard_index, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; const size_bytes = placement.localShape(self._shape).byteSize(); const destination = sub_slice.data()[0..size_bytes]; - const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, destination); + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); @@ -320,9 +322,9 @@ pub const Buffer = struct { var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); defer shard_slice.free(allocator); - for (self._shards.constSlice(), 0..) |shard, shard_index| { + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { const sub_slice = placement.shardSlice(shard_index, slice); - const maybe_event = try shard.toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); @@ -419,41 +421,6 @@ test "device round-trip" { } } -test "a buffer on a sub-mesh has one shard per device of that mesh" { - const io = std.testing.io; - const allocator = std.testing.allocator; - // Four devices: the accelerator's when it has them, else a CPU platform's - // (none on a host without the CPU plugin, such as aarch64 Linux). - var devices: testing.Devices = undefined; - devices.init(allocator, io, 4) catch return error.SkipZigTest; - defer devices.deinit(); - const platform = devices.platform; - // Two of the four devices, out of the platform's order. - const nodes = [_]Sharding.PhysicalNode{ .device(platform.devices[3]), .device(platform.devices[1]) }; - var physical: Sharding.PhysicalMesh = try .fromTree(allocator, platform.target, .axis(.bus, .tree, &nodes)); - defer physical.deinit(allocator); - const mesh: Sharding.Mesh = try .init("pair", &physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); - - var values: [8]u32 = undefined; - for (&values, 0..) |*value, i| value.* = @intCast(i); - const shape = Shape.init(.{ .b = 4, .d = 2 }, .u32).withPartitioning(&mesh, .{ .b = .model }); - var buffer: Buffer = try .fromBytes(io, platform, shape, std.mem.sliceAsBytes(&values)); - defer buffer.deinit(); - try testing.expectShards(allocator, io, &buffer, std.mem.sliceAsBytes(&values), .default); - - const read: Slice = try buffer.toSliceAlloc(allocator, io); - defer read.free(allocator); - try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); - var into: [8]u32 = undefined; - try buffer.toSlice(io, .init(shape, std.mem.sliceAsBytes(&into))); - try std.testing.expectEqualSlices(u32, &values, &into); - - const pinned: Buffer.HostAccessible = try .init(allocator, io, platform, shape, .undonatable); - defer pinned.deinit(allocator); - pinned.view(u32).copyFrom(&values); - try testing.expectShards(allocator, io, &pinned.buffer, std.mem.sliceAsBytes(&values), .host_pinned); -} - pub const PjrtBufferMetadata = struct { shape: Shape, ty: pjrt.BufferType, @@ -479,16 +446,10 @@ pub const PjrtBufferMetadata = struct { } }; -/// The `kind` memory of `device`, a device of a buffer's mesh, which names it -/// by its index in `platform.devices`. Outside `Buffer` for `Buffer.HostAccessible`. -pub fn deviceMemory(platform: *const Platform, device: Sharding.Device, kind: Memory.Kind) *const Memory { - stdx.debug.assert(device.id < platform.devices.len, "Mesh device {d} is not one of the {d} devices of {t}", .{ device.id, platform.devices.len, platform.target }); - return platform.devices[device.id].memory(kind) orelse stdx.debug.panic("Device {d} doesn't have {t} memory", .{ device.id, kind }); -} - fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { - const placed = platform.placedShape(shape); - return Sharding.Placement.init(&placed) catch |err| { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(&sh) catch |err| { @branchHint(.cold); std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index 20d90657..b3e82a9c 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -17,7 +17,6 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; -const deviceMemory = @import("../buffer.zig").deviceMemory; const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; const Platform = @import("../platform.zig").Platform; const Shape = @import("../shape.zig").Shape; @@ -250,7 +249,7 @@ pub fn init( ._shards = .empty, }; const metadata: PjrtBufferMetadata = .init(platform, shape_); - const devices = metadata.placement.mesh().?.devicesInCanonicalOrder(); + const devices = platform.physical_mesh.devices_in_canonical_order; const api = platform.pjrt_api; var allocations: HostPinnedAllocations = .empty; @@ -260,7 +259,7 @@ pub fn init( errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); for (devices, 0..) |device, shard_index| { - const memory = deviceMemory(platform, device, .host_pinned).pjrt_memory; + const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; diff --git a/zml/exe.zig b/zml/exe.zig index 179df29f..32c0ac48 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -310,7 +310,7 @@ pub const Exe = struct { /// Non-donated inputs remain owned by the caller. pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); - self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); + self.exe.call(io, self.args, &self.results, opts); self.exe.deinitDonatedInputs(self.args); self.results.fill(output_refs); } @@ -334,9 +334,7 @@ pub const Exe = struct { /// Executes with optional waiting. The caller owns all input handles, including donated inputs. /// Donated input contents cannot be used after the call; output buffers have their own handles. /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. - /// A failed launch writes no outputs; with `wait`, a failed execution destroys the outputs it - /// wrote, so `results_` holds no buffer after any error. - pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) pjrt.ApiError!void { + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, .arg_count = arguments.expected_shapes.len, @@ -344,7 +342,6 @@ pub const Exe = struct { }); defer span.end(); - const api = self.platform.pjrt_api; var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; @@ -353,7 +350,7 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, }; - try self.exe.execute(api, .{ + self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, @@ -361,22 +358,40 @@ pub const Exe = struct { // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, - }); - - // Every partition's event is destroyed, awaited first when waiting. - var failure: ?pjrt.ApiError = null; - if (events_slice) |slice| for (slice) |e| { - const ev = e orelse continue; - if (opts.wait) ev.await(api, io) catch |err| { - failure = failure orelse err; - }; - ev.deinit(api); + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - if (failure) |err| { - for (results_.flat_buffers.raw_buffers) |buffer| buffer.deinit(api); - return err; + + switch (self.platform.target) { + .neuron => { + for (events_slice.?) |e| { + if (e) |ev| { + if (opts.wait) { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + } + ev.deinit(self.platform.pjrt_api); + } + } + }, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + ev.deinit(self.platform.pjrt_api); + } + } + }, } } + + pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { + const exe = try self.exe.executable(self.platform.pjrt_api); + return try exe.getCompiledMemoryStats(self.platform.pjrt_api); + } }; /// A typed executable whose input and output buffer structures are derived from @@ -460,7 +475,7 @@ pub fn FnExe(comptime function_: anytype) type { /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); - self.exe.call(io, self.args, &self.results, opts) catch |err| std.debug.panic("PJRT execution failed with: {}", .{err}); + self.exe.call(io, self.args, &self.results, opts); self.exe.deinitDonatedInputs(self.args); self.results.fill(call.outputs); } @@ -610,7 +625,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); - try exe.call(io, args, &results, .{ .wait = wait }); + exe.call(io, args, &results, .{ .wait = wait }); var output = results.get(Buffer); defer output.deinit(); @@ -618,8 +633,6 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { for (value_buffer._shards.constSlice()) |shard| { try std.testing.expect(shard.isDeleted(platform.pjrt_api)); } - // Executing over the deleted input is an error, not a panic. - try std.testing.expect(std.meta.isError(exe.call(io, args, &results, .{ .wait = true }))); value_buffer.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); diff --git a/zml/io.zig b/zml/io.zig index 837b1f1e..ab8d8e6e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,32 +1,1743 @@ -//! Checkpoint loading and virtual filesystem integration. The user guide is -//! docs/learn/loader.md; each file's header holds its internals: -//! -//! - io/loader.zig: options and their checks, the set-up task, starts, -//! program compiles and groups, admission, and the one check of every -//! binding before the backends see it (`LoadSpec`). -//! - io/backend.zig: backend selection, the request size rule and the -//! `LoadSpec` contract. -//! - io/TensorStore.zig, io/Source.zig: bindings, views, and the sources -//! that describe a tensor's checkpoint bytes. -//! - io/direct_loader.zig: sizing, per-file planning, FIFO claims, source -//! workers, per-device pumps and transfer completion. -//! - io/ShardGrid.zig: where a destination's logical bytes land. -//! - io/buffered_loader.zig: whole-tensor staging and positional reads. -//! - io/dma_calibration.zig: DMA block selection. -//! - io/host_memory.zig: pinned host arenas and their blocks. -//! - io/log.zig: log scopes. Loader test support is in `zml.testing`. -//! - vfs: local and remote files behind one `std.Io`, the remote error -//! contract, retries and the rate-limit hold (vfs/request.zig). - const std = @import("std"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); pub const VFS = @import("vfs"); -pub const Loader = @import("io/loader.zig").Loader; -pub const TensorStore = @import("io/TensorStore.zig"); +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("zml.zig").Bufferized; +const CreateOptions = @import("platform.zig").CreateOptions; +const Exe = @import("exe.zig").Exe; +const mem = @import("mem.zig"); +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); +const safetensors = @import("safetensors.zig"); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Placement = Sharding.Placement; +const Slice = @import("slice.zig").Slice; +const Tensor = @import("tensor.zig").Tensor; -pub const dma_calibration = @import("io/dma_calibration.zig"); +const log = std.log.scoped(.@"zml/io"); -test { - std.testing.refAllDecls(@This()); +pub const TensorStore = struct { + pub const Binding = struct { + tensors: []*safetensors.Tensor, + transformed: bool, + memory: Memory.Kind = .default, + }; + + registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, + id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .registry = registry, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), + .id_to_sources = .empty, + .allocator = allocator, + .arena = arena, + }; + } + + pub fn deinit(self: *TensorStore) void { + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); + } + + fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { + const gop = try self.id_to_sources.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Id {} already has associated sources", .{id}); + } + errdefer self.id_to_sources.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = sources; + } + + fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { + const entry = self.getPtrFromKey(key) orelse return null; + + const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); + copy.* = entry.*; + + return copy; + } + + fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { + const sources = self.id_to_sources.get(id) orelse return null; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + return sources.tensors[0]; + } + + pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + return self.registry.reader(io, key, buffer); + } + + pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { + const sources = self.id_to_sources.get(id) orelse return error.NotFound; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + + return sources.tensors[0].reader(io, buffer, .{}); + } + + pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); + } + + pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { + const entry_ptr = self.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn view(self: *TensorStore) View { + return .{ .store = self }; + } + + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + + pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + pub fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); + } + + const resolved_mesh = self.store.mesh(mesh_); + if (@TypeOf(partitioning) == Sharding.Partitioning) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); + } + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| + .parse(p_tags, resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); + } + + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const source = self.store.dupeSource(key) orelse return null; + + const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); + errdefer self.store.arena.allocator().free(sources); + sources[0] = source; + + var shape = source.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + + return tensor; + } + + /// Accepts "null" as tags + fn parseTags(tags: anytype) ?[]const Shape.Tag { + return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); + } + + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { + const arena = self.store.arena.allocator(); + + var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); + defer tensor_list.deinit(arena); + + var buffer: [256]u8 = undefined; + for (sources) |subkey| { + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const tensor = self.store.dupeSource(key) orelse return null; + tensor_list.appendAssumeCapacity(tensor); + } + + const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; + errdefer arena.free(tensors); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + + return tensor; + } + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getShape(key); + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + }; + return self.store.getShape(key); + } + + pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + var key_buffer: [256]u8 = undefined; + const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getReader(key, io, buffer); + } + + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix() orelse ""; + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } + }; +}; + +pub const Loader = struct { + allocator: std.mem.Allocator, + platform: *const Platform, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + pinned_buffer_pools: []mem.DynamicBufferPool, + group: stdx.Io.LimitedGroup, + bytes_loaded: std.atomic.Value(usize) = .init(0), + delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, + + pub const Opts = struct { + pub const default: Opts = .{ + .parallelism = 1, + .dma_chunks = 2, + .dma_chunk_size = 4096, + }; + parallelism: usize, + dma_chunks: usize, + dma_chunk_size: usize, + }; + + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { + const pool_count = platform.devices.len; + const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); + errdefer allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(allocator, device); + } + + const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); + errdefer allocator.free(buffer_pools); + for (buffer_pools) |*pool_| { + pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); + } + errdefer for (buffer_pools, 0..) |*pool_, i| { + pool_.deinit(dma_allocators[i].allocator()); + }; + + return .{ + .allocator = allocator, + .platform = platform, + .dma_allocators = dma_allocators, + .dma_chunk_size = opts.dma_chunk_size, + .pinned_buffer_pools = buffer_pools, + .group = .init(opts.parallelism), + }; + } + + pub fn deinit(self: *Loader) void { + self.delivered.deinit(self.allocator); + for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); + self.allocator.free(self.pinned_buffer_pools); + self.allocator.free(self.dma_allocators); + } + + pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { + return self.group.await(io); + } + + pub const LoadOpts = struct { + progress: ?*std.Progress.Node = null, + }; + + pub const LoadError = error{TransformedTensorNotDelivered}; + + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { + const tensor_count = meta.count(Tensor, model); + + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(buffers, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { + flattened_buffers_[i] = buffer; + } + }.call, .{flattened_buffers}); + + const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { + flattened_tensors_[i] = tensor; + } + }.call, .{flattened_tensors}); + + // Refuse before scheduling anything, so a failure leaves no task in flight. + var missing: usize = 0; + for (flattened_tensors) |tensor| { + const sources = store.getSourcesById(tensor.id) orelse continue; + if (!sources.transformed or self.delivered.contains(tensor.id)) { + continue; + } + + missing += 1; + logNotDelivered(arena.allocator(), tensor, sources.tensors); + } + + if (missing != 0) { + return error.TransformedTensorNotDelivered; + } + + for (flattened_tensors, flattened_buffers) |tensor, buffer| { + if (store.getSourcesById(tensor.id)) |sources| { + if (sources.transformed) continue; + } + + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); + } + } + + fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { + const max_names = 8; + var names: std.Io.Writer.Allocating = .init(arena); + for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { + names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; + } + + if (sources.len > max_names) { + names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; + } + + log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); + } + + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); + } + + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { + const sources = store.getSourcesById(tensor.id) orelse { + std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); + return; + }; + stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); + + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { + log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); + unreachable; + }; + } + + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { + log.err("Failed to load tensor {s}: {}", .{ source.name, e }); + loaded.* = false; + return; + }; + loaded.* = true; + } + + fn loadSingleInner( + self: *Loader, + io: std.Io, + source: *safetensors.Tensor, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + opts: LoadOpts, + ) !void { + var reader = try source.reader(io, &.{}, .{}); + defer reader.deinit(); + + var writer = try MemoryWriter.init( + self.allocator, + io, + self.platform, + self.pinned_buffer_pools, + self.dma_allocators, + self.dma_chunk_size, + shape, + buffer, + memory, + ); + defer writer.deinit(self.allocator); + + const scale = 1024; + + if (opts.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + writer.setProgress(&node); + defer writer.setProgress(null); + var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); + const total = try reader.interface.streamRemaining(&progress_writer.interface); + try progress_writer.interface.flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } else { + const total = try reader.interface.streamRemaining(writer.interface()); + try writer.interface().flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } + } + + pub fn loadExecute( + self: *Loader, + arena: std.mem.Allocator, + io: std.Io, + tensor: Tensor, + buffer: *Buffer, + store: *const TensorStore, + exe: *const Exe, + opts: LoadOpts, + ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { + const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; + const buffers = try arena.alloc(Buffer, sources.len); + const loaded = try arena.alloc(bool, sources.len); + @memset(loaded, false); + defer for (buffers, loaded) |*b, l| if (l) b.deinit(); + + var node = if (opts.progress) |progress| p: { + var writer = std.Io.Writer.Allocating.init(arena); + print: { + writer.writer.writeAll("Running executable on ") catch break :print; + for (sources, 0..) |source, i| { + writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; + } + } + break :p progress.start(writer.written(), 1); + } else null; + defer if (node) |*n| n.end(); + + for (sources, 0..) |source, i| { + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); + } + self.group.await(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + }; + + if (std.mem.findScalar(bool, loaded, false)) |_| { + return error.LoadFailed; + } + + var args = try exe.args(arena); + var results = try exe.results(arena); + + args.set(.{buffers}); + exe.call(io, args, &results, .{ .wait = true }); + + buffer.* = results.get(Buffer); + try self.delivered.put(self.allocator, tensor.id, {}); + } +}; + +test "Loader.loadExecute cleans up donated and non-donated inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = @import("testing.zig").env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); + defer loader.deinit(); + + inline for (.{ false, true }) |donate| { + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + const sum = value.add(weight); + return if (donate) sum.reuseBuffer(value) else sum; + } + }; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); + defer exe.deinit(); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); + + var output: Buffer = undefined; + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expect(loader.delivered.contains(tensor.id)); + } +} + +pub const ProgressWriter = struct { + inner: *std.Io.Writer, + progress: *std.Progress.Node, + interface: std.Io.Writer, + total: usize = 0, + scale: usize, + + pub const InitOpts = struct { + scale: usize = 1, + }; + + pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { + return .{ + .inner = inner_, + .progress = progress_, + .scale = opts.scale, + .interface = .{ + .buffer = inner_.buffer, + .end = inner_.end, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + .sendFile = sendFile, + }, + }, + }; + } + + pub fn pre(self: *ProgressWriter) usize { + self.inner.buffer = self.interface.buffer; + self.inner.end = self.interface.end; + return self.inner.end; + } + + pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { + self.interface.buffer = self.inner.buffer; + self.interface.end = self.inner.end; + const drained_pre = len_pre -| self.interface.end; + self.total += drained_pre + total; + self.progress.setCompletedItems(self.total / self.scale); + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.drain(self.inner, data, splat); + self.post(len_pre, total); + return total; + } + + pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); + self.post(len_pre, total); + return total; + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.flush(self.inner); + } + + pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.rebase(self.inner, preserve, capacity); + } +}; + +pub const MemoryWriter = union(enum) { + direct: DirectMemoryWriter, + buffered: BufferedMemoryWriter, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !MemoryWriter { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .furiosa => .{ + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), + }, + .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) + else + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, + }; + } + + pub fn interface(self: *MemoryWriter) *std.Io.Writer { + return switch (self.*) { + .direct => &self.direct.interface, + .buffered => &self.buffered.interface, + }; + } + + pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { + switch (self.*) { + .direct => self.direct.deinit(), + .buffered => self.buffered.deinit(allocator), + } + } + + pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { + switch (self.*) { + .direct => self.direct.setProgress(progress), + .buffered => {}, + } + } +}; + +pub const BufferedMemoryWriter = struct { + io: std.Io, + platform: *const Platform, + shape: Shape, + buffer: *Buffer, + interface: std.Io.Writer, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { + return .{ + .io = io, + .platform = platform, + .shape = shape, + .buffer = buffer, + .interface = .{ + .buffer = try allocator.alloc(u8, shape.byteSize()), + .vtable = &.{ + .drain = std.Io.Writer.fixedDrain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { + if (self.interface.buffer.len > 0) { + allocator.free(self.interface.buffer); + } + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + self.buffer.* = Buffer.from( + self.io, + self.platform, + self.shape, + @ptrCast(self.interface.buffer), + .{ .wait = true }, + ) catch return std.Io.Writer.Error.WriteFailed; + } +}; + +const DirectShardWriter = struct { + const EventContext = struct { + self: *DirectShardWriter, + err: ?*pjrt.Error = null, + pjrt_event: *pjrt.Event, + event: std.Io.Event = .unset, + buffer: []u8, + }; + + allocator: std.mem.Allocator, + io: std.Io, + memory: *const Memory, + pool: *mem.DynamicBufferPool, + total: usize, + pjrt_buffer: *pjrt.Buffer, + + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + offset: usize = 0, + + interface: std.Io.Writer, + flip_flop: u1 = 0, + events_contexts: [2]?EventContext = @splat(null), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { + const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); + const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( + memory.platform.pjrt_api, + .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, + }, + ); + errdefer transfer_manager.deinit(memory.platform.pjrt_api); + + const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); + errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); + + const buf = try pool.get(allocator, io); + + return .{ + .allocator = allocator, + .io = io, + .memory = memory, + .pool = pool, + .total = shape.byteSize(), + .pjrt_buffer = pjrt_buffer, + .transfer_manager = transfer_manager, + .interface = .{ + .buffer = buf[0..@min(buf.len, shape.byteSize())], + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectShardWriter) void { + self.transfer_manager.deinit(self.memory.platform.pjrt_api); + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + + const chunk = data[0]; + if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; + + const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; + if (needs_fresh_buffer) { + try self.interface.flush(); + } + + @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); + self.interface.end += chunk.len; + + const buffer_full = self.interface.end == self.interface.buffer.len; + if (buffer_full) { + try self.interface.flush(); + } + + return chunk.len; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + try self.flushBuffered(); + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + try self.interface.flush(); + } + + fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { + if (self.offset >= self.total) return; + + const pjrt_api = self.memory.platform.pjrt_api; + + const current_buffer = self.interface.buffer; + const buffered = self.interface.buffered(); + + const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; + if (slice.len == 0) return; + const is_last = (self.offset + slice.len) >= self.total; + + const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { + log.err("error when transferring data to device: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + + const ctx = &self.events_contexts[@intCast(self.flip_flop)]; + ctx.* = .{ + .self = self, + .buffer = current_buffer, + .pjrt_event = transfer_event, + }; + + transfer_event.onReady(pjrt_api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); + ctx_.err = err; + ctx_.event.set(ctx_.self.io); + } + }.call, &(ctx.*.?)) catch |err| { + log.err("error when setting up transfer completion callback: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + + if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { + defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; + ctx_previous.event.waitUncancelable(self.io); + defer ctx_previous.pjrt_event.deinit(pjrt_api); + if (ctx_previous.err) |e| { + defer e.deinit(pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; + } + } + + if (is_last) { + defer ctx.* = null; + defer self.interface = .failing; + const ctx_ = &ctx.*.?; + ctx_.event.waitUncancelable(self.io); + defer ctx_.pjrt_event.deinit(pjrt_api); + if (ctx_.err) |e| { + defer e.deinit(pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; + } + } else { + self.interface.end = 0; + const buf = self.pool.get(self.allocator, self.io) catch |err| { + log.err("unable to get a new buffer from the pool: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; + } + self.flip_flop ^= 1; + self.offset += slice.len; + } +}; + +const ShardProgress = struct { + const scale: usize = 1024; + + node: std.Progress.Node, + label: [32]u8 = undefined, + completed: usize = 0, + + fn set(self: *ShardProgress, completed: usize) void { + self.completed = completed; + self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); + } +}; + +// Dispatch planning bridges two different orders: +// +// 1. Placement traversal emits byte ranges per shard writer, in device order. +// That order is convenient for asking "what bytes belong to this shard?", +// but it does not match how a tensor file is read. +// +// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter +// must therefore consume dispatch spans in increasing global byte offset. +// +// We first collect placement spans, sort them by global byte range, then fold +// identical ranges into one primary dispatch span plus mirror writers. Identical +// ranges represent replicated shard data: the reader provides those bytes once, +// the primary writer receives them zero-copy, and mirrors receive a copy. +const DispatchSpans = struct { + const DispatchSpan = struct { + start: usize, + end: usize, + primary_writer: usize, + mirror_writer_start: usize, + mirror_writer_len: usize, + }; + + const PlacementSpan = struct { + writer_index: usize, + start: usize, + len: usize, + order: usize, + }; + + spans: []DispatchSpan, + mirror_writers: []usize, + + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(&shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); + + var placement_span_count: usize = 0; + for (0..ordered_devices.len) |device_index| { + placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); + } + + var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); + defer placement_spans.deinit(allocator); + + const byte_strides = shape.computeByteStrides(); + + for (0..ordered_devices.len) |writer_index| { + appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); + } + + std.debug.assert(placement_spans.items.len == placement_span_count); + + var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); + errdefer spans.deinit(allocator); + + var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); + errdefer mirror_writers.deinit(allocator); + + try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); + + const spans_ = try spans.toOwnedSlice(allocator); + errdefer allocator.free(spans_); + + const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); + errdefer allocator.free(mirror_writers_); + + return .{ + .spans = spans_, + .mirror_writers = mirror_writers_, + }; + } + + fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { + allocator.free(self.spans); + allocator.free(self.mirror_writers); + } + + fn deduplicateByRange( + allocator: std.mem.Allocator, + placement_spans: []PlacementSpan, + total_bytes: usize, + spans: *std.ArrayList(DispatchSpan), + mirror_writers: *std.ArrayList(usize), + ) !void { + const SortContext = struct { + fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { + if (lhs.start != rhs.start) return lhs.start < rhs.start; + if (lhs.len != rhs.len) return lhs.len < rhs.len; + return lhs.order < rhs.order; + } + }; + + std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); + + var i: usize = 0; + var cursor: usize = 0; + while (i < placement_spans.len) { + const span = placement_spans[i]; + if (span.start != cursor) return error.NonContiguousShardPlacement; + + const mirror_writer_start = mirror_writers.items.len; + var j = i + 1; + while (j < placement_spans.len) : (j += 1) { + const mirror = placement_spans[j]; + if (mirror.start != span.start or mirror.len != span.len) break; + try mirror_writers.append(allocator, mirror.writer_index); + } + + try spans.append(allocator, .{ + .start = span.start, + .end = span.start + span.len, + .primary_writer = span.writer_index, + .mirror_writer_start = mirror_writer_start, + .mirror_writer_len = j - i - 1, + }); + cursor += span.len; + i = j; + } + + if (cursor != total_bytes) return error.NonContiguousShardPlacement; + } + + fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { + placement_spans.appendAssumeCapacity(.{ + .writer_index = writer_index, + .start = start, + .len = len, + .order = placement_spans.items.len, + }); + } + + fn appendShardPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + shape: Shape, + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + ) void { + if (shape.rank() == 0) { + appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); + return; + } + + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); + } + + fn appendShardAxisPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + axis: usize, + contiguous_axis: usize, + base_start: i64, + ) void { + const slice = slices[axis]; + if (slice.size == 0) return; + + if (axis == contiguous_axis) { + const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); + const span_len: usize = @intCast(slice.size * byte_strides[axis]); + appendPlacementSpan(placement_spans, writer_index, span_start, span_len); + return; + } + + var i: i64 = 0; + while (i < slice.size) : (i += 1) { + const child_start = base_start + (slice.start + i) * byte_strides[axis]; + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); + } + } + + fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { + if (shape.rank() == 0) return 1; + + const contiguous_axis = contiguousSliceAxis(shape, slices); + var count: usize = 1; + for (slices[0..contiguous_axis]) |slice| { + count *= @intCast(slice.size); + } + return count; + } + + fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { + var axis = shape.rank() - 1; + while (axis > 0) { + const slice = slices[axis]; + if (slice.start != 0 or slice.size != shape.dim(axis)) break; + axis -= 1; + } + return axis; + } +}; + +// Direct load writer state machine. +// +// std.Io.Reader +// | +// v +// DirectMemoryWriter.interface.buffer +// | aliases active primary DirectShardWriter DMA buffer +// v +// commitWindow() +// |-- primary: advance DirectShardWriter.interface.end +// |-- mirrors: copy committed bytes with writeAll() +// |-- boundaries: flush full shard buffers / chunk fence +// v +// DirectShardWriter.flushBuffered() +// | +// v +// PJRT AsyncHostToDeviceTransferManager +// +// The public writer never owns a host staging buffer: it exposes a visible +// prefix of a real shard DMA buffer. The visible prefix may extend beyond the +// current dispatch span to coalesce reader requests, but `commitWindow` must +// scatter those bytes before submitting or rotating the active buffer. The +// first active-primary segment is zero-copy; crossed spans and mirrors treat +// the active buffer as scratch and copy into their own shard writers. +// +// `byte_cursor` is the global tensor position. Shard writer `interface.end` is +// local to the current shard DMA buffer. +pub const DirectMemoryWriter = struct { + allocator: std.mem.Allocator, + shard_writers: []DirectShardWriter, + // Global stream-order spans produced from placement; this is the routing table for committed bytes. + dispatch_spans: DispatchSpans, + // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. + span_index: usize = 0, + // Global tensor byte offset already scattered into shard writers. + byte_cursor: usize = 0, + // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. + active_writer_index: usize, + // Start offset of new reader bytes inside the active shard writer buffer. + window_start: usize, + // Logical maximum public alias window before forcing a cross-shard flush fence. + dma_chunk_size: usize, + shard_progress: ?[]ShardProgress = null, + interface: std.Io.Writer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !DirectMemoryWriter { + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(&placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); + var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); + errdefer allocator.free(shard_writers); + + var initialized: usize = 0; + errdefer for (shard_writers[0..initialized]) |*writer| { + writer.deinit(); + }; + + var pjrt_buffers: Buffer.Shards = .empty; + for (ordered_devices, 0..) |device, i| { + const pool = &pools[device.id]; + const shard_dma_allocator = dma_allocators[device.id].allocator(); + const pjrt_mem = platform.devices[device.id].memory(memory).?; + + shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); + initialized += 1; + + pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); + } + + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); + + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); + errdefer dispatch_spans.deinit(allocator); + + const first_span = dispatch_spans.spans[0]; + const first_writer = &shard_writers[first_span.primary_writer]; + const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); + + return .{ + .allocator = allocator, + .shard_writers = shard_writers, + .dispatch_spans = dispatch_spans, + .active_writer_index = first_span.primary_writer, + .window_start = first_writer.interface.end, + .dma_chunk_size = dma_chunk_size, + .interface = .{ + .buffer = first_writer.interface.buffer[0..first_window], + .end = 0, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectMemoryWriter) void { + self.setProgress(null); + for (self.shard_writers) |*writer| { + writer.deinit(); + } + self.allocator.free(self.shard_writers); + self.dispatch_spans.deinit(self.allocator); + } + + pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { + const parent = progress orelse { + const states = self.shard_progress orelse return; + for (states) |*s| { + s.node.end(); + } + self.allocator.free(states); + self.shard_progress = null; + return; + }; + + std.debug.assert(self.shard_progress == null); + const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; + for (states, self.shard_writers, 0..) |*state, writer, i| { + state.completed = 0; + const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; + const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; + state.node = parent.start(label, total_items); + state.node.setCompletedItems(0); + } + self.shard_progress = states; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + try self.commitWindow(); + if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + + const consumed = consumed: { + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + const copied = self.writeWindow(bytes); + n += copied; + + if (copied < bytes.len) break :consumed n; + } + + const pattern = data[data.len - 1]; + switch (pattern.len) { + 0 => {}, + 1 => { + const dest = self.interface.buffer[self.interface.end..]; + const count = @min(dest.len, splat); + @memset(dest[0..count], pattern[0]); + + self.interface.end += count; + n += count; + }, + else => for (0..splat) |_| { + const copied = self.writeWindow(pattern); + n += copied; + + if (copied < pattern.len) break; + }, + } + + break :consumed n; + }; + + try self.commitWindow(); + return consumed; + } + + fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { + const dest = self.interface.buffer[self.interface.end..]; + const n = @min(dest.len, bytes.len); + @memcpy(dest[0..n], bytes[0..n]); + self.interface.end += n; + return n; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + // Commit the active public window if one remains + if (self.span_index < self.dispatch_spans.spans.len) { + try self.commitWindow(); + } + + for (self.shard_writers, 0..) |*shard_writer, i| { + try shard_writer.interface.flush(); + if (self.shard_progress) |states| { + states[i].set(shard_writer.total); + } + } + + self.interface = .failing; + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + + try self.commitWindow(); + if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; + } + + // `interface.buffer` is a clipped alias of the active primary shard DMA + // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; + // this function scatters that visible range through dispatch spans. + // + // The leading segment for `active_writer_index` is already in the right + // DMA buffer and is committed by advancing that shard writer. Later spans + // use the same bytes as scratch and are copied before any full buffers are + // submitted or the public alias is rotated. + fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + // All dispatch spans are consumed; no shard DMA buffer remains valid to + // expose as the public alias until flush marks the writer as failing. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Only bytes after `window_start` are new reader writes. Bytes before + // it may already be pending in the active shard writer. + const source = self.interface.buffer[self.window_start..self.interface.end]; + if (source.len == 0) return; + + // The visible alias window may cross many dispatch spans. Walk the + // global cursor and route each subrange to its primary and mirrors. + var consumed: usize = 0; + while (consumed < source.len) { + if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; + + const span = self.dispatch_spans.spans[self.span_index]; + if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; + + const n = @min(source.len - consumed, span.end - self.byte_cursor); + const chunk = source[consumed..][0..n]; + + // The first span in the active alias is already in the primary DMA + // buffer. Later ranges use the active buffer as scratch and must be + // copied into their real destination writer. + if (span.primary_writer == self.active_writer_index and consumed == 0) { + self.shard_writers[span.primary_writer].interface.end += n; + } else { + const primary_writer = &self.shard_writers[span.primary_writer]; + if (span.primary_writer == self.active_writer_index) { + @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); + primary_writer.interface.end += n; + } else { + try primary_writer.interface.writeAll(chunk); + } + } + if (self.shard_progress) |states| { + states[span.primary_writer].set(states[span.primary_writer].completed + n); + } + + const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; + for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { + const mirror_writer = &self.shard_writers[mirror_writer_index]; + if (mirror_writer_index == self.active_writer_index) { + @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); + mirror_writer.interface.end += n; + } else { + try mirror_writer.interface.writeAll(chunk); + } + if (self.shard_progress) |states| { + states[mirror_writer_index].set(states[mirror_writer_index].completed + n); + } + } + + self.byte_cursor += n; + consumed += n; + if (self.byte_cursor == span.end) { + self.span_index += 1; + } + } + + // All scratch copies are complete now, so full DMA buffers may be + // submitted without invalidating bytes still needed for scatter. + for (self.shard_writers) |*writer| { + if (writer.interface.end == writer.interface.buffer.len) { + try writer.interface.flush(); + } + } + + if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { + for (self.shard_writers) |*writer| { + try writer.interface.flush(); + } + } + + // The last committed window may finish the tensor exactly. Leave an + // empty public window so any further writes fail through std.Io. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Rotate the public alias to the primary writer for the next stream + // span. `window_start` preserves that writer's already-buffered prefix + // so only newly advanced bytes are scattered on the next commit. + const next_span = self.dispatch_spans.spans[self.span_index]; + const next_writer = &self.shard_writers[next_span.primary_writer]; + self.active_writer_index = next_span.primary_writer; + self.window_start = next_writer.interface.end; + + // Publish the next public alias. Its backing memory is the next primary + // shard buffer, but the visible length is allowed to coalesce across + // upcoming spans until a buffer or tensor boundary. + const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; + const buffer_remaining = next_writer.interface.buffer.len - self.window_start; + const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); + const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; + const tensor_remaining = total - self.byte_cursor; + const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); + if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; + + self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; + self.interface.end = self.window_start; + } +}; + +const DirectMemoryWriterDeviceTest = struct { + const WriteMode = enum { + stream_remaining, + writable_slice_greedy, + }; + + pub const Scenario = struct { + name: []const u8, + create_options: CreateOptions, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + write_mode: WriteMode = .stream_remaining, + writable_slice_min_len: usize = 128, + pool_chunks: usize = 4, + pool_chunk_size: usize = 1 << 20, + memory: Memory.Kind = .default, + }; + + allocator: std.mem.Allocator, + io: std.Io, + + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { + var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; + defer platform.deinit(self.allocator, self.io); + + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + std.log.warn("mesh: {f}", .{mesh}); + try self.runDirectMemoryWriter( + platform, + scenario.shape.withPartitioning(&mesh, partitioning), + scenario.write_mode, + scenario.writable_slice_min_len, + scenario.pool_chunks, + scenario.pool_chunk_size, + scenario.memory, + ); + } + + fn runDirectMemoryWriter( + self: DirectMemoryWriterDeviceTest, + platform: *const Platform, + shape: Shape, + write_mode: WriteMode, + writable_slice_min_len: usize, + pool_chunks: usize, + pool_chunk_size: usize, + memory: Memory.Kind, + ) !void { + const slice = try Slice.alloc(self.allocator, shape); + defer slice.free(self.allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = @as(f32, @floatFromInt(i)); + } + + const pool_count = platform.devices.len; + const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); + defer self.allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(self.allocator, device); + } + + const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); + defer self.allocator.free(pools); + for (pools) |*pool| { + pool.* = .init(pool_chunks, pool_chunk_size); + } + defer for (pools, 0..) |*pool, i| { + pool.deinit(dma_allocators[i].allocator()); + }; + + var written_buffer: Buffer = undefined; + var writer: DirectMemoryWriter = try .init( + self.allocator, + self.io, + platform, + pools, + dma_allocators, + pool_chunk_size, + shape, + &written_buffer, + memory, + ); + defer writer.deinit(); + defer written_buffer.deinit(); + + switch (write_mode) { + .stream_remaining => { + var reader: std.Io.Reader = .fixed(slice.constData()); + const streamed = try reader.streamRemaining(&writer.interface); + try std.testing.expectEqual(slice.constData().len, streamed); + }, + .writable_slice_greedy => { + var offset: usize = 0; + while (offset < slice.constData().len) { + const min_len = @max(@as(usize, 1), writable_slice_min_len); + const dest = try writer.interface.writableSliceGreedy(min_len); + const to_write = @min(dest.len, slice.constData().len - offset); + if (to_write == 0) return std.Io.Writer.Error.WriteFailed; + @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); + writer.interface.advance(to_write); + offset += to_write; + } + }, + } + + try writer.interface.flush(); + try written_buffer.await(self.io); + + var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); + defer written_slice.free(self.allocator); + try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); + } +}; + +test "DirectMemoryWriter: replicated tensor" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "replicated_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "batch_model_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + }, .{ .batch = .batch, .model = .model }); +} + +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_folded_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + }, .{ .model = .model }); +} + +test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto_writable_slice", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .write_mode = .writable_slice_greedy, + .writable_slice_min_len = 64, + .pool_chunk_size = 1024, + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 3D topology folded model + replicated batch" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "topology_3d_folded_model", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .cpu = .{ .device_count = 8 }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + }, .{ .batch = .replicated, .model = .model }); +} + +test "MemoryWriter can produce a host pinned buffer" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "host_pinned", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .memory = .host_pinned, + }, .{ .batch = .replicated, .model = .model }); } diff --git a/zml/io/ShardGrid.zig b/zml/io/ShardGrid.zig deleted file mode 100644 index 41969285..00000000 --- a/zml/io/ShardGrid.zig +++ /dev/null @@ -1,352 +0,0 @@ -//! Where a destination's logical bytes land. Its devices hold the boxes of an -//! even grid over the tensor (`Sharding.Placement` splits every axis into -//! equal shards), each box packed row-major on its devices, and devices -//! holding the same box are one writer set. Box order and logical order agree -//! on a box's elements, so a contiguous logical range lands as one contiguous -//! range of every box it touches: ranges are placed by arithmetic, never by -//! listing rows. -//! -//! For a [4, 8] tensor whose columns are split in two, each letter one -//! element: -//! -//! Logical bytes Box 0 Box 1 -//! a b c d e f g h a b c d e f g h -//! i j k l m n o p i j k l m n o p -//! q r s t u v w x q r s t u v w x -//! y z A B C D E F y z A B C D E F -//! -//! The logical range from c to n lands as c..l on box 0 and e..n on box 1. -//! Runs are the aligned pieces that stay in one box (`run`: a shard of the -//! innermost split axis, four elements here); a worker walks them to place -//! a range's bytes, the planner only counts them. - -const std = @import("std"); - -const constants = @import("../constants.zig"); -const Platform = @import("../platform.zig").Platform; -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); - -const ShardGrid = @This(); - -const max_rank = constants.MAX_RANK; -/// Every box has at least one device, and a writer mask holds 64. -pub const max_boxes = 64; - -rank: usize, -/// Elements per axis, of the tensor and of one box. -dims: [max_rank]u64 = @splat(1), -shard: [max_rank]u64 = @splat(1), -/// Byte strides, logical and inside a box. -strides: [max_rank]u64 = @splat(0), -box_strides: [max_rank]u64 = @splat(0), -/// A box's index is row-major over the shard counts of the axes. -index_strides: [max_rank]u64 = @splat(0), -element: u64, -total: u64, -box_bytes: u64, -/// Aligned runs of `run` logical bytes each land in one box, contiguous. -run: u64, -box_count: usize, -/// The canonical device indices holding each box. -writers: [max_boxes]u64 = @splat(0), -/// One box: the shape each device holds. -local: Shape, -/// The devices in canonical order, which the `writers` masks index; -/// borrowed from the placed shape's mesh (the platform's when `shape` names -/// none). -devices: []const Sharding.Device, - -/// The grid of `shape` placed on `platform` (`Platform.placedShape`), its -/// sub-byte storage packed (`Shape.packedShape`): what PJRT allocates. -/// `shape` must be non-empty and place on the platform's devices, as the -/// loader's front end checked. -pub fn init(platform: *const Platform, shape: Shape) ShardGrid { - const placed = platform.placedShape(shape).packedShape(); - std.debug.assert(placed.byteSize() != 0); - const placement = Sharding.Placement.init(&placed) catch |err| - std.debug.panic("ShardGrid: {f} does not place, though checked: {s}", .{ placed, @errorName(err) }); - const local = placement.localShape(placed); - const devices = placement.mesh().?.devicesInCanonicalOrder(); - std.debug.assert(devices.len <= max_boxes); - const element: u64 = placed.dtype().sizeOf(); - var grid: ShardGrid = .{ - .rank = placed.rank(), - .element = element, - .total = placed.byteSize(), - .box_bytes = local.byteSize(), - .run = placed.byteSize(), - .box_count = 1, - .local = local, - .devices = devices, - }; - const strides = placed.computeByteStrides(); - var box_stride = element; - var index_stride: u64 = 1; - var axis = grid.rank; - while (axis > 0) { - axis -= 1; - const dim: u64 = @intCast(placed.dim(axis)); - const shard: u64 = @intCast(local.dim(axis)); - grid.dims[axis] = dim; - grid.shard[axis] = shard; - grid.strides[axis] = @intCast(strides.get(axis)); - grid.box_strides[axis] = box_stride; - grid.index_strides[axis] = index_stride; - box_stride *= shard; - const count = dim / shard; - index_stride *= count; - // The innermost split axis cuts the runs. - if (count > 1 and grid.run == grid.total) grid.run = shard * grid.strides[axis]; - } - // A placement holds every box on at least one of its devices. - std.debug.assert(index_stride <= devices.len); - grid.box_count = @intCast(index_stride); - for (0..devices.len) |writer| { - const slices = placement.slices(writer, placed); - var box: u64 = 0; - for (slices.constSlice(), 0..) |slice, i| box += @as(u64, @intCast(slice.start)) / grid.shard[i] * grid.index_strides[i]; - grid.writers[@intCast(box)] |= @as(u64, 1) << @intCast(writer); - } - for (grid.writers[0..grid.box_count]) |mask| std.debug.assert(mask != 0); - return grid; -} - -fn coordinate(self: *const ShardGrid, offset: u64, axis: usize) u64 { - return offset / self.strides[axis] % self.dims[axis]; -} - -/// The box holding logical byte `offset`, and where the byte lands inside it. -pub fn locate(self: *const ShardGrid, offset: u64) struct { usize, u64 } { - var box: u64 = 0; - var at = offset % self.element; - for (0..self.rank) |i| { - const c = self.coordinate(offset, i); - box += c / self.shard[i] * self.index_strides[i]; - at += c % self.shard[i] * self.box_strides[i]; - } - return .{ @intCast(box), at }; -} - -/// The bytes of box `box` before logical offset `offset`: a logical range -/// `[start, end)` lands on that box at `[rankIn(start), rankIn(end))`. -/// Counted axis by axis, as in a lexicographic comparison. -pub fn rankIn(self: *const ShardGrid, box: usize, offset: u64) u64 { - if (offset >= self.total) return self.box_bytes; - var before: u64 = 0; - for (0..self.rank) |i| { - const c = self.coordinate(offset, i); - const first = @as(u64, box) / self.index_strides[i] % (self.dims[i] / self.shard[i]) * self.shard[i]; - before += @min(c -| first, self.shard[i]) * self.box_strides[i]; - if (c < first or c >= first + self.shard[i]) return before; - } - return before + offset % self.element; -} - -/// The end of the run holding logical byte `offset`, at most `limit`. -pub fn runEnd(self: *const ShardGrid, offset: u64, limit: u64) u64 { - return @min(limit, (offset / self.run + 1) * self.run); -} - -/// The runs with a byte in `[start, end)`, `start < end`. -pub fn runCount(self: *const ShardGrid, start: u64, end: u64) u64 { - return (end + self.run - 1) / self.run - start / self.run; -} - -/// Devices per box: the grid is even, so every box has as many. -pub fn replicas(self: *const ShardGrid) usize { - return @popCount(self.writers[0]); -} - -const testing = struct { - const CreateOptions = @import("../platform.zig").CreateOptions; - - const Scenario = struct { - name: []const u8, - device_count: u32, - physical_mesh: CreateOptions.PhysicalMesh = .auto, - shape: Shape, - logical_mesh: Sharding.LogicalMesh, - strategy: Sharding.Strategy, - /// Logical ranges are cut every `step` bytes, unaligned to rows. - step: u64, - }; - - fn run(scenario: Scenario, partitioning: anytype) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - var platform = Platform.init(allocator, io, .cpu, .{ - .physical_mesh = scenario.physical_mesh, - .cpu = .{ .device_count = scenario.device_count }, - }) catch return error.SkipZigTest; - defer platform.deinit(allocator, io); - const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); - try expectPlaces(allocator, platform, scenario.shape.withPartitioning(&mesh, partitioning), scenario.step); - } - - /// Walks the runs of every `step`-byte range into the writers' boxes and - /// compares with each device's slices, element by element; checks that - /// `rankIn` bounds the range's bytes on each box and that `runCount` - /// counts the runs. - fn expectPlaces(allocator: std.mem.Allocator, platform: *const Platform, shape: Shape, step: u64) !void { - const grid: ShardGrid = .init(platform, shape); - const placed = platform.placedShape(shape).packedShape(); - const placement: Sharding.Placement = try .init(&placed); - const box_bytes: usize = @intCast(grid.box_bytes); - const total: usize = @intCast(grid.total); - const source = try allocator.alloc(u8, total); - defer allocator.free(source); - for (source, 0..) |*byte, i| byte.* = @truncate(i *% 131 +% 17); - - const expected = try allocator.alloc(u8, grid.devices.len * box_bytes); - defer allocator.free(expected); - const element: usize = @intCast(grid.element); - for (0..grid.devices.len) |writer| { - const slices = placement.slices(writer, placed); - var index: [max_rank]u64 = @splat(0); - var cursor: usize = 0; - while (cursor < box_bytes) : (cursor += element) { - var logical: u64 = 0; - for (0..grid.rank) |i| logical += (@as(u64, @intCast(slices.get(i).start)) + index[i]) * grid.strides[i]; - @memcpy(expected[writer * box_bytes + cursor ..][0..element], source[@intCast(logical)..][0..element]); - var axis = grid.rank; - while (axis > 0) { - axis -= 1; - index[axis] += 1; - if (index[axis] < grid.shard[axis]) break; - index[axis] = 0; - } - } - } - - const actual = try allocator.alloc(u8, expected.len); - defer allocator.free(actual); - @memset(actual, 0); - var start: u64 = 0; - while (start < grid.total) : (start += step) { - const end = @min(grid.total, start + step); - var low: [max_boxes]u64 = @splat(std.math.maxInt(u64)); - var high: [max_boxes]u64 = @splat(0); - var runs: u64 = 0; - var at = start; - while (at < end) : (runs += 1) { - const stop = grid.runEnd(at, end); - const box, const landing = grid.locate(at); - low[box] = @min(low[box], landing); - high[box] = @max(high[box], landing + stop - at); - var mask = grid.writers[box]; - while (mask != 0) : (mask &= mask - 1) { - const writer: usize = @ctz(mask); - @memcpy(actual[writer * box_bytes + @as(usize, @intCast(landing)) ..][0..@intCast(stop - at)], source[@intCast(at)..@intCast(stop)]); - } - at = stop; - } - try std.testing.expectEqual(runs, grid.runCount(start, end)); - for (0..grid.box_count) |box| { - const first = grid.rankIn(box, start); - const last = grid.rankIn(box, end); - if (high[box] == 0) { - try std.testing.expectEqual(first, last); - } else { - try std.testing.expectEqual(low[box], first); - try std.testing.expectEqual(high[box], last); - } - } - } - try std.testing.expectEqualSlices(u8, expected, actual); - } -}; - -test "a grid places replicated tensors, across range boundaries" { - try testing.run(.{ - .name = "replicated", - .device_count = 4, - .shape = Shape.init(.{ .rows = 9, .cols = 257 }, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - .step = 773, - }, .{ .rows = .replicated, .cols = .replicated }); -} - -test "a grid places packed sub-byte storage" { - const logical = Shape.init(.{ .rows = 9, .cols = 256 }, .u2); - try testing.run(.{ - .name = "packed_u2", - .device_count = 4, - .shape = logical, - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - .step = 131, - }, .{ .rows = .replicated, .cols = .x }); -} - -test "a grid places 1D mirrored and folded sharding" { - try testing.run(.{ - .name = "mirrored_1d", - .device_count = 4, - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .shape = Shape.init(.{ .rows = 7, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .step = 2053, - }, .{ .rows = .replicated, .model = .model }); - try testing.run(.{ - .name = "folded_1d", - .device_count = 4, - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .shape = Shape.init(.{ .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; - }, - .step = 3001, - }, .{ .model = .model }); -} - -test "a grid places 2D and 3D sharding" { - try testing.run(.{ - .name = "batch_model_2d", - .device_count = 4, - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - .step = 4093, - }, .{ .batch = .batch, .model = .model }); - try testing.run(.{ - .name = "folded_model_3d", - .device_count = 8, - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; - }, - .step = 8191, - }, .{ .batch = .replicated, .model = .model }); - // Three axes, the middle and the last split: runs are a quarter row. - try testing.run(.{ - .name = "middle_and_last_3d", - .device_count = 4, - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .shape = Shape.init(.{ .a = 3, .b = 4, .c = 8 }, .u16), - .logical_mesh = .mesh(.{ .x = .low_bandwidth, .y = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x, .y = .link_y }), - .step = 37, - }, .{ .a = .replicated, .b = .x, .c = .y }); -} - -test "a grid places a scalar on every device" { - try testing.run(.{ - .name = "scalar", - .device_count = 4, - .shape = Shape.init(.{}, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - .step = 3, - }, .{}); -} diff --git a/zml/io/Source.zig b/zml/io/Source.zig deleted file mode 100644 index ea90111a..00000000 --- a/zml/io/Source.zig +++ /dev/null @@ -1,1349 +0,0 @@ -//! A tensor the loader assembles from checkpoint bytes by moving them, never -//! by computing: a checkpoint tensor, or slices, reshapes, transposes, -//! broadcasts, paddings, concatenations and stacks of checkpoint tensors. -//! The ops are `zml.Tensor`'s shape ops, with the same names, tags and -//! argument conventions; slices are static and contiguous, and padding is -//! zeros only. -//! -//! A source comes from its store (`View.source`) and its pieces live in -//! the store's arena, where the ops that rewrite them allocate. Nothing is -//! freed before `TensorStore.deinit`, and the store must not move while -//! sources point to it. The ops touch neither the registry nor the -//! bindings. Sources of different stores do not mix: `concatenate`, -//! `stack` and `bindProgram` panic on them. Placement comes last: -//! `withPartitioning` and `as` return a `Placed`, which has no ops and -//! drops its placement with `unplaced`. A `Placed` is what binds: -//! `Placed.bind` records the model tensor the loader fills from it, -//! `bindProgram` one a program computes from several. Sources are built -//! during model init. -const std = @import("std"); - -const stdx = @import("stdx"); - -const constants = @import("../constants.zig"); -const DataType = @import("../dtype.zig").DataType; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; -const backend = @import("backend.zig"); -const TensorStore = @import("TensorStore.zig"); - -const Source = @This(); - -store: *TensorStore, -/// Always replicated: placement comes last (`Placed`). -_shape: Shape, -/// Disjoint lattices of elements read from checkpoint tensors; elements no -/// piece covers are zero. zml/io's own: a binding keeps them (`Placed`) -/// and the loader lowers them to bytes (`Placed.lower`). -_pieces: []const Piece, - -/// Checkpoint bits read through an affine lattice: for every `j` with -/// `j[k] < dims[k].count`, the element at row-major index -/// `origin + sum(j[k] * dims[k].step)` of the source's shape is the element -/// at bit `offset + sum(j[k] * dims[k].stride)` of `tensor`. Reshapes keep -/// the row-major order, so they never touch a piece. -const Piece = struct { - tensor: *const safetensors.Tensor, - offset: u64, - origin: u64, - /// Outermost first, none of count 1. - dims: []const Dim, -}; - -const Dim = struct { - count: u64, - /// Row-major elements of the source's shape between two steps. - step: u64, - /// Bits of the checkpoint tensor between two steps; zero repeats them. - stride: u64, -}; - -/// `pad`'s per-axis options, as for `Tensor.pad`. -pub const Pad = Tensor.Pad; - -/// The checkpoint tensor `subkey` of `view`, with `tags` (null keeps the -/// checkpoint's); null when the checkpoint has no such tensor. -pub fn init(view: TensorStore.View, subkey: []const u8, tags: ?[]const Shape.Tag) ?Source { - const tensor = view.get(subkey) orelse return null; - const store = view.store; - var shape_ = tensor.shape; - if (tags) |user_tags| { - stdx.debug.assert(user_tags.len == shape_.rank(), "checkpoint tensor {s} has shape {f}, but got {d} tags: {f}", .{ tensor.name, shape_, user_tags.len, stdx.fmt.stringsZ(user_tags) }); - @memcpy(shape_._tags.slice(), user_tags); - } - shape_._sharding = .replicated; - const arena = store.arena.allocator(); - const elements: u64 = shape_.count(); - const pieces: []const Piece = if (elements == 0) &.{} else blk: { - const dims_: []const Dim = if (elements == 1) - &.{} - else - arena.dupe(Dim, &.{.{ .count = elements, .step = 1, .stride = shape_.dtype().bitSizeOf() }}) catch oom(); - break :blk arena.dupe(Piece, &.{.{ .tensor = tensor, .offset = 0, .origin = 0, .dims = dims_ }}) catch oom(); - }; - return .{ .store = store, ._shape = shape_, ._pieces = pieces }; -} - -pub fn shape(self: Source) Shape { - return self._shape; -} - -pub fn dim(self: Source, axis_: anytype) i64 { - return self._shape.dim(axis_); -} - -pub fn dims(self: *const Source) []const i64 { - return self._shape.dims(); -} - -pub fn dtype(self: Source) DataType { - return self._shape.dtype(); -} - -pub fn rank(self: Source) u4 { - return self._shape.rank(); -} - -pub fn axis(self: Source, axis_: anytype) u3 { - return self._shape.axis(axis_); -} - -pub fn count(self: Source) usize { - return self._shape.count(); -} - -pub fn format(self: Source, writer: *std.Io.Writer) !void { - try writer.print("Source({f}, {d} pieces)", .{ self._shape, self._pieces.len }); -} - -// Reshapes: the row-major order is unchanged, so are the pieces. - -pub fn withTags(self: Source, tags: anytype) Source { - return self.reshaped(self._shape.withTags(tags)); -} - -pub fn rename(self: Source, renames: anytype) Source { - return self.reshaped(self._shape.rename(renames)); -} - -pub fn renameTag(self: Source, old: anytype, new: anytype) Source { - return self.reshaped(self._shape.setTag(old, new)); -} - -pub fn splitAxis(self: Source, ax: anytype, split_shape: anytype) Source { - return self.reshaped(self._shape.splitAxis(ax, split_shape)); -} - -pub fn merge(self: Source, merges: anytype) Source { - return self.reshaped(self._shape.mergeAxes(merges)); -} - -pub fn reshape(self: Source, output_shape: anytype) Source { - return self.reshaped(self._shape.reshape(output_shape)); -} - -pub fn flatten(self: Source) Source { - return self.reshape(.{self.count()}); -} - -pub fn insertAxes(self: Source, axis_: anytype, tags: anytype) Source { - const tags_ = Shape.parseTags(tags); - const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); - var res = self._shape; - const ones: [constants.MAX_RANK]i64 = @splat(1); - res._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; - return self.reshaped(res); -} - -pub fn appendAxes(self: Source, tags: anytype) Source { - return self.insertAxes(.last, tags); -} - -pub fn squeeze(self: Source, axis_: anytype) Source { - const a = self.axis(axis_); - stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); - return self.reshaped(self._shape.remove(a)); -} - -fn reshaped(self: Source, new_shape: Shape) Source { - stdx.debug.assert(new_shape.count() == self.count(), "Source reshape from {f} to {f} changes the element count", .{ self._shape, new_shape }); - var res = self; - res._shape = new_shape.withDtype(self.dtype()); - res._shape._sharding = .replicated; - return res; -} - -/// Reinterprets the bits as `dt`, like `Tensor.bitCast`: to a narrower type, -/// each element becomes the elements of a new last axis `.bitcast`; to a -/// wider one, the last axis must hold one element of `dt`. -pub fn bitCast(self: Source, dt: DataType) Source { - const src_bits: u64 = self.dtype().bitSizeOf(); - const tgt_bits: u64 = dt.bitSizeOf(); - if (src_bits == tgt_bits) return .{ .store = self.store, ._shape = self._shape.withDtype(dt), ._pieces = self._pieces }; - var ctx: Rewrite = .init(self.store); - defer ctx.deinit(); - if (src_bits > tgt_bits) { - const ratio = std.math.divExact(u64, src_bits, tgt_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); - const res_shape = self._shape.append(.{ .bitcast = @as(i64, @intCast(ratio)) }).withDtype(dt); - for (self._pieces) |piece| { - var flat: Flat = .fromPiece(piece); - flat.origin *= ratio; - for (flat.dims.slice()) |*d| d.step *= ratio; - flat.dims.append(.{ .count = ratio, .step = 1, .stride = tgt_bits }) catch tooManyDims(); - ctx.emitFlat(flat); - } - return ctx.finish(res_shape); - } - const ratio = std.math.divExact(u64, tgt_bits, src_bits) catch std.debug.panic("bitCast from {t} to {t}: bit sizes are not multiples", .{ self.dtype(), dt }); - const last: u3 = @intCast(self.rank() - 1); - stdx.debug.assert(self.dim(last) == @as(i64, @intCast(ratio)), "bitCast from {f} to {t} expects a last axis of {d} elements", .{ self._shape, dt, ratio }); - const res_shape = self._shape.remove(last).withDtype(dt); - for (self._pieces) |piece| { - for (ctx.decompose(self._shape, piece)) |*axis_piece| { - // The elements merged into one must be one contiguous run of bits. - const inner: ?usize = for (axis_piece.dims.constSlice(), 0..) |d, k| { - if (d.axis == last) break k; - } else null; - const whole = axis_piece.origin[last] == 0 and inner != null and blk: { - const d = axis_piece.dims.get(inner.?); - break :blk d.count == ratio and d.step == 1 and d.stride == src_bits; - }; - if (!whole) std.debug.panic("bitCast of {f} to {t}: an element of {t} would gather bits that are not contiguous in the checkpoint", .{ self._shape, dt, dt }); - _ = axis_piece.dims.orderedRemove(inner.?); - ctx.emit(res_shape, axis_piece.*); - } - } - return ctx.finish(res_shape); -} - -// Ops that move elements. - -/// Like `Tensor.slice`, for a static, contiguous slice (`step` 1). -pub fn slice(self: Source, axis_: anytype, s: Tensor.Slice) Source { - stdx.debug.assert(s.dyn_start == null, "Source.slice takes static slices, got {f}", .{s}); - stdx.debug.assert(s.step == 1, "Source.slice takes contiguous slices, got {f}", .{s}); - const a = self.axis(axis_); - const range = s.absolute(self.dim(a)); - const lo: u64 = @intCast(range.start); - const hi: u64 = @intCast(range.end); - const res_shape = self._shape.setDim(a, range.end - range.start); - var ctx: Rewrite = .init(self.store); - defer ctx.deinit(); - for (self._pieces) |piece| { - for (ctx.decompose(self._shape, piece)) |axis_piece| ctx.clip(res_shape, axis_piece, a, lo, hi); - } - const sliced = ctx.finish(res_shape); - return if (s.singleton) sliced.reshaped(res_shape.remove(a)) else sliced; -} - -/// Like `Tensor.split`; the result lives in the store's arena. -pub fn split(self: Source, axis_: anytype, split_sizes: []const i64) []Source { - const a = self.axis(axis_); - var total: i64 = 0; - for (split_sizes) |n| total += n; - stdx.debug.assert(total == self.dim(a), "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ total, self.dim(a) }); - const res = self.store.arena.allocator().alloc(Source, split_sizes.len) catch oom(); - var start: i64 = 0; - for (split_sizes, res) |n, *part| { - part.* = self.slice(a, .{ .start = start, .end = start + n }); - start += n; - } - return res; -} - -pub fn chunkExact(self: Source, axis_: anytype, comptime n_chunks: comptime_int) [n_chunks]Source { - const a = self.axis(axis_); - const size = @divExact(self.dim(a), n_chunks); - var chunks: [n_chunks]Source = undefined; - for (&chunks, 0..) |*chunk, i| { - const start = @as(i64, @intCast(i)) * size; - chunk.* = self.slice(a, .{ .start = start, .end = start + size }); - } - return chunks; -} - -/// Like `Tensor.transpose`: no axes reverses them. -pub fn transpose(self: Source, axes_: anytype) Source { - const given = self._shape.axes(axes_); - const r = self.rank(); - var permutation: [constants.MAX_RANK]u3 = undefined; - if (given.len == 0) { - for (0..r) |i| permutation[i] = @intCast(r - 1 - i); - } else { - stdx.debug.assert(given.len == r, "transpose expects {d} axes for {f}, got {d}", .{ r, self._shape, given.len }); - @memcpy(permutation[0..r], given.constSlice()); - } - var as_i64: [constants.MAX_RANK]i64 = undefined; - for (permutation[0..r], as_i64[0..r]) |p, *q| q.* = p; - const res_shape = self._shape.transpose(as_i64[0..r]); - // Axes of one element move nothing: when the others keep their order, - // the row-major order is unchanged, so are the pieces. - var previous: ?u3 = null; - const moves = for (permutation[0..r]) |p| { - if (self.dim(p) == 1) continue; - if (previous) |q| if (p < q) break true; - previous = p; - } else false; - if (!moves) return self.reshaped(res_shape); - var inverse: [constants.MAX_RANK]u3 = undefined; - for (permutation[0..r], 0..) |p, i| inverse[p] = @intCast(i); - var ctx: Rewrite = .init(self.store); - defer ctx.deinit(); - for (self._pieces) |piece| { - for (ctx.decompose(self._shape, piece)) |axis_piece| { - var moved = axis_piece; - for (0..r) |i| moved.origin[i] = axis_piece.origin[permutation[i]]; - for (moved.dims.slice()) |*d| d.axis = inverse[d.axis]; - ctx.emit(res_shape, moved); - } - } - return ctx.finish(res_shape); -} - -/// Like `Tensor.pad`, with zero as the only padding value. -pub fn pad(self: Source, padding_value: anytype, paddings: anytype) Source { - stdx.debug.assert(isZero(padding_value), "Source.pad pads with zeros, got {any}", .{padding_value}); - const options = self._shape.parseAxesOptions(Tensor.Pad, paddings, .{}); - var res_shape = self._shape; - var moves = false; - for (options.constSlice(), 0..) |p, a| { - stdx.debug.assert(p.low >= 0 and p.high >= 0 and p.interior >= 0, "Source.pad cannot crop, got {any} on axis {d}", .{ p, a }); - const d = self.dim(a); - res_shape._dims.set(a, p.low + d + @max(d - 1, 0) * p.interior + p.high); - if (p.low != 0 or p.interior != 0 or (a != 0 and p.high != 0)) moves = true; - } - // Rows appended after the last one move no element. - if (!moves) return .{ .store = self.store, ._shape = res_shape, ._pieces = self._pieces }; - var ctx: Rewrite = .init(self.store); - defer ctx.deinit(); - for (self._pieces) |piece| { - for (ctx.decompose(self._shape, piece)) |*axis_piece| { - for (options.constSlice(), 0..) |p, a| { - axis_piece.origin[a] = @as(u64, @intCast(p.low)) + axis_piece.origin[a] * @as(u64, @intCast(p.interior + 1)); - } - for (axis_piece.dims.slice()) |*d| d.step *= @intCast(options.get(d.axis).interior + 1); - ctx.emit(res_shape, axis_piece.*); - } - } - return ctx.finish(res_shape); -} - -/// Like `Tensor.broad`: repeats the elements along the axes `other` adds or -/// grows from 1. The loader reads a few large repetitions once each, and -/// copies many small ones from one read of the source per job. -pub fn broad(self: Source, other: Shape) Source { - stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self._shape, other }); - if (std.mem.eql(i64, self.dims(), other.dims())) return self; - const r = self.rank(); - var axes_map: [constants.MAX_RANK]u3 = undefined; - if (r == 0 or r == other.rank()) { - for (0..r) |i| axes_map[i] = @intCast(i); - } else { - for (self._shape.tags(), 0..) |t, i| axes_map[i] = other.axis(t); - } - const res_shape = other.withDtype(self.dtype()); - // The axes the elements repeat along: new ones and those grown from 1. - var repeated: [constants.MAX_RANK]bool = @splat(true); - for (0..r) |i| { - if (self.dim(i) != 1) repeated[axes_map[i]] = false; - } - var ctx: Rewrite = .init(self.store); - defer ctx.deinit(); - for (self._pieces) |piece| { - for (ctx.decompose(self._shape, piece)) |axis_piece| { - var moved: AxisPiece = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset }; - for (0..r) |i| moved.origin[axes_map[i]] = axis_piece.origin[i]; - for (axis_piece.dims.constSlice()) |d| { - var mapped = d; - mapped.axis = axes_map[d.axis]; - moved.dims.append(mapped) catch tooManyDims(); - } - for (0..other.rank()) |b| { - if (!repeated[b] or other.dim(b) == 1) continue; - moved.dims.append(.{ .count = @intCast(other.dim(b)), .axis = @intCast(b), .step = 1, .stride = 0 }) catch tooManyDims(); - } - ctx.emit(res_shape, moved); - } - } - return ctx.finish(res_shape); -} - -/// Like `Tensor.concatenate`; the sources must share every dimension but -/// `axis_`, and the dtype. -pub fn concatenate(sources: []const Source, axis_: anytype) Source { - stdx.debug.assert(sources.len > 0, "concatenate expects at least one source", .{}); - if (sources.len == 1) return sources[0]; - const first = sources[0]; - const a = first.axis(axis_); - var total: i64 = 0; - for (sources) |s| { - stdx.debug.assert(s.store == first.store, "concatenate expects sources of one store", .{}); - const same = s.dtype() == first.dtype() and s.rank() == first.rank() and for (0..first.rank()) |i| { - if (i != a and s.dim(i) != first.dim(i)) break false; - } else true; - stdx.debug.assert(same, "concatenate along axis {d} expects matching shapes, got {f} and {f}", .{ a, first._shape, s._shape }); - total += s.dim(a); - } - const res_shape = first._shape.setDim(a, total); - const outer_stride = elementStrides(res_shape)[a]; - var ctx: Rewrite = .init(first.store); - defer ctx.deinit(); - var offset: u64 = 0; - for (sources) |s| { - for (s._pieces) |piece| { - if (a == 0) { - // Along the outermost axis, a source's elements follow the previous one's. - var flat: Flat = .fromPiece(piece); - flat.origin += offset * outer_stride; - ctx.emitFlat(flat); - continue; - } - for (ctx.decompose(s._shape, piece)) |*axis_piece| { - axis_piece.origin[a] += offset; - ctx.emit(res_shape, axis_piece.*); - } - } - offset += @intCast(s.dim(a)); - } - return ctx.finish(res_shape); -} - -/// Like `Tensor.stack`; the sources must have the same shape and tags. -pub fn stack(sources: []const Source, axis_: anytype, tag: anytype) Source { - stdx.debug.assert(sources.len > 0, "stack expects at least one source", .{}); - const first = sources[0]; - for (sources[1..]) |s| { - stdx.debug.assert(s.store == first.store, "stack expects sources of one store", .{}); - stdx.debug.assert(first._shape.eqlWithTags(s._shape), "stack expects source shapes to match, got {f} and {f}", .{ first._shape, s._shape }); - } - const a: u3 = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) @intCast(first.rank()) else first.axis(axis_); - const res_shape = first._shape.insertTag(axis_, @intCast(sources.len), tag); - const r = first.rank(); - var ctx: Rewrite = .init(first.store); - defer ctx.deinit(); - for (sources, 0..) |s, i| { - for (s._pieces) |piece| { - if (a == 0) { - var flat: Flat = .fromPiece(piece); - flat.origin += i * first.count(); - ctx.emitFlat(flat); - continue; - } - for (ctx.decompose(s._shape, piece)) |axis_piece| { - var moved = axis_piece; - std.mem.copyBackwards(u64, moved.origin[a + 1 .. r + 1], axis_piece.origin[a..r]); - moved.origin[a] = i; - for (moved.dims.slice()) |*d| { - if (d.axis >= a) d.axis += 1; - } - ctx.emit(res_shape, moved); - } - } - } - return ctx.finish(res_shape); -} - -/// This source placed on `mesh_` by `partitioning`: a struct keyed by the -/// source's tags (axes it does not name are replicated), or a -/// `Sharding.Partitioning`. `.replicated` places it on every device without -/// a mesh, and takes only `.{}`. Placement comes last: a `Placed` only -/// binds (`Placed.bind`, `bindProgram`) or drops its placement -/// (`Placed.unplaced`). -pub fn withPartitioning(self: Source, comptime mesh_: @EnumLiteral(), partitioning: anytype) Placed { - var placed = self._shape; - if (mesh_ == .replicated) { - const T = @TypeOf(partitioning); - if (comptime !(stdx.meta.isStruct(T) and std.meta.fieldNames(T).len == 0)) - @compileError("Source.withPartitioning(.replicated, ...) takes .{}: a replicated source has no partitioning, got " ++ @typeName(T)); - } else { - placed = self._shape.withPartitioning(self.store.mesh(mesh_), partitioning); - } - return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; -} - -/// This source in `placed`, a shape built elsewhere: its dimensions, tags -/// and sharding. Like `reshape`, it moves no element: `placed` has the -/// source's element count and dtype. Placement comes last, as for -/// `withPartitioning`. -pub fn as(self: Source, placed: Shape) Placed { - stdx.debug.assert(placed.dtype() == self.dtype() and placed.count() == self.count(), "Source {f} cannot be placed as {f}", .{ self._shape, placed }); - return .{ .store = self.store, .shape = placed, ._pieces = self._pieces }; -} - -/// Makes a model tensor of `output` that `program` computes while loading, -/// `program.forward(output, inputs)` over the placed sources `inputs` in this -/// order (`TensorStore.Program.init`). The loader lands each input as it -/// lands a bound source, placed by its shape, in default device memory; -/// compiles the program once for every tensor of an equal program over the -/// same placed shapes; runs it, and frees the inputs. `forward` returns a -/// tensor of `output`, placed alike, in default device memory. -pub fn bindProgram(inputs: []const Placed, output: Shape, program: anytype) Tensor { - stdx.debug.assert(inputs.len != 0, "Source.bindProgram: a program computing {f} needs an input", .{output}); - const store = inputs[0].store; - for (inputs) |input| { - stdx.debug.assert(input.store == store, "Source.bindProgram: the inputs of {f} come from different stores", .{output}); - } - const arena = store.arena.allocator(); - const kept = arena.dupe(Placed, inputs) catch oom(); - const erased = TensorStore.Program.init(arena, program) catch oom(); - return bindNew(store, output, .{ .compute = .{ .program = erased, .inputs = kept } }); -} - -/// A source with its placement (`withPartitioning`, `as`): the only thing -/// that binds, and what a binding keeps (`TensorStore.Binding`). -pub const Placed = struct { - store: *TensorStore, - /// The placed shape, whose row-major elements the pieces index. - shape: Shape, - /// See `Source._pieces`. - _pieces: []const Piece, - - /// Makes the model tensor the loader fills from this source, in default - /// memory unless `View.setMemory` names another. The binding keeps the - /// source as it is; the loader lowers it to bytes (`lower`). - pub fn bind(self: Placed) Tensor { - return bindNew(self.store, self.shape, .{ .load = .{ .source = self } }); - } - - /// This source without its placement, for code that takes a placed - /// source and reworks it by its placement (a layout per device), then - /// places the result again. - pub fn unplaced(self: Placed) Source { - var shape_ = self.shape; - shape_._sharding = .replicated; - return .{ .store = self.store, ._shape = shape_, ._pieces = self._pieces }; - } - - /// The checkpoint tensor this source is, whole and alone, if it is: one - /// piece that reads all of its bits in order. - pub fn whole(self: Placed) ?*const safetensors.Tensor { - if (self._pieces.len != 1) return null; - const piece = self._pieces[0]; - if (piece.offset != 0 or piece.origin != 0 or piece.tensor.byteSize() != self.shape.byteSize()) return null; - const contiguous = switch (piece.dims.len) { - 0 => self.shape.count() == 1, - 1 => piece.dims[0].count == self.shape.count() and piece.dims[0].step == 1 and piece.dims[0].stride == self.shape.dtype().bitSizeOf(), - else => false, - }; - return if (contiguous) piece.tensor else null; - } - - /// The bytes the loader moves, one part per piece in piece order, in - /// `arena`; none for a source that reads no bytes. The one rule from a - /// source's bits to bytes: each part's dimensions outermost first, the - /// innermost run that is contiguous on both sides as its width. - /// `error.SubBytePosition` when a run starts or ends inside a byte. - pub fn lower(self: Placed, arena: std.mem.Allocator) error{ OutOfMemory, SubBytePosition }![]backend.LoadPart { - const parts = try arena.alloc(backend.LoadPart, self._pieces.len); - for (parts, self._pieces) |*part, piece| part.* = try lowerPiece(arena, self.shape, piece); - return parts; - } -}; - -/// A new tensor of `shape_`, bound to `binding`: the one maker of -/// `store`'s bindings (`Placed.bind`, `bindProgram`), whose pieces are -/// disjoint by construction: the loader relies on that and does not check. -fn bindNew(store: *TensorStore, shape_: Shape, binding: TensorStore.Binding) Tensor { - const tensor: Tensor = .fromShape(shape_); - store.id_to_sources.putNoClobber(store.allocator, tensor.id, binding) catch oom(); - return tensor; -} - -/// `piece` of a source placed in `shape_` as one part (`Placed.lower`), its -/// dimensions allocated with `allocator`, none for one run. -fn lowerPiece(allocator: std.mem.Allocator, shape_: Shape, piece: Piece) error{ OutOfMemory, SubBytePosition }!backend.LoadPart { - const ebits: u64 = shape_.dtype().bitSizeOf(); - var lowered: stdx.BoundedArray(backend.Lattice.Dim, backend.max_part_dims) = .empty; - for (piece.dims) |d| lowered.appendAssumeCapacity(.{ .count = d.count, .src_stride = d.stride, .dst_stride = d.step * ebits }); - var width: u64 = ebits; - if (lowered.len != 0) { - const inner = lowered.get(lowered.len - 1); - if (inner.dst_stride == ebits and inner.src_stride == ebits) { - width = inner.count * ebits; - _ = lowered.pop(); - } - } - const dst_bits = piece.origin * ebits; - var aligned = piece.offset % 8 == 0 and dst_bits % 8 == 0 and width % 8 == 0; - for (lowered.slice()) |*d| { - aligned = aligned and d.src_stride % 8 == 0 and d.dst_stride % 8 == 0; - d.src_stride /= 8; - d.dst_stride /= 8; - } - if (!aligned) return error.SubBytePosition; - return .{ .source = piece.tensor, .lattice = .{ - .src_offset = piece.offset / 8, - .dst_offset = dst_bits / 8, - .width = width / 8, - .dims = if (lowered.len == 0) &.{} else try allocator.dupe(backend.Lattice.Dim, lowered.constSlice()), - } }; -} - -// The machinery. A piece is kept flat; an op that addresses axes rewrites it -// in coordinates of the input shape (`AxisPiece`, every dimension moving one -// axis), moves the coordinates, then flattens it in the result's shape. - -/// A piece while an op rewrites it, its dimensions inline. -const Flat = struct { - tensor: *const safetensors.Tensor, - offset: u64, - origin: u64, - dims: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty, - - fn fromPiece(piece: Piece) Flat { - var flat: Flat = .{ .tensor = piece.tensor, .offset = piece.offset, .origin = piece.origin }; - flat.dims.appendSlice(piece.dims) catch tooManyDims(); - return flat; - } - - /// The same lattice with dimension `k` cut to `n` steps from step `first`. - fn sub(self: Flat, k: usize, first: u64, n: u64) Flat { - var res = self; - const d = self.dims.get(k); - res.origin += first * d.step; - res.offset += first * d.stride; - res.dims.buffer[k].count = n; - return res; - } -}; - -/// A piece in coordinates of a shape: the element at -/// `origin + sum(j[k] * dims[k].step along dims[k].axis)`. -const AxisPiece = struct { - tensor: *const safetensors.Tensor, - offset: u64, - origin: [constants.MAX_RANK]u64 = @splat(0), - dims: stdx.BoundedArray(AxisDim, backend.max_part_dims) = .empty, -}; - -const AxisDim = struct { - count: u64, - axis: u3, - step: u64, - stride: u64, -}; - -/// The scratch of one op and the pieces it makes. -const Rewrite = struct { - store: *TensorStore, - scratch: std.heap.ArenaAllocator, - work: std.ArrayList(Flat) = .empty, - /// The current piece in coordinates of its shape (`decompose`). - axis_pieces: std.ArrayList(AxisPiece) = .empty, - out: std.ArrayList(Piece) = .empty, - - fn init(store: *TensorStore) Rewrite { - return .{ .store = store, .scratch = .init(store.allocator) }; - } - - fn deinit(self: *Rewrite) void { - self.scratch.deinit(); - } - - fn finish(self: *Rewrite, res_shape: Shape) Source { - var s = res_shape; - s._sharding = .replicated; - return .{ - .store = self.store, - ._shape = s, - ._pieces = self.store.arena.allocator().dupe(Piece, self.out.items) catch oom(), - }; - } - - /// `piece` in coordinates of `shape_`, cut where a dimension would carry - /// from one axis into the next; valid until the next call. - fn decompose(self: *Rewrite, shape_: Shape, piece: Piece) []AxisPiece { - const scratch = self.scratch.allocator(); - self.axis_pieces.clearRetainingCapacity(); - self.work.append(scratch, .fromPiece(piece)) catch oom(); - const strides = elementStrides(shape_); - while (self.work.pop()) |flat| { - switch (fitAxes(shape_, strides, flat)) { - .piece => |axis_piece| self.axis_pieces.append(scratch, axis_piece) catch oom(), - .overflow => |overflowing| self.cut(shape_, strides, flat, overflowing), - } - } - return self.axis_pieces.items; - } - - /// Splits `flat`, which carries past the end of axis `a`, into pieces - /// that do not: at the row boundaries of `a` when its longest dimension - /// there steps regularly through the rows, else one piece per step. - fn cut(self: *Rewrite, shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat, a: u3) void { - const scratch = self.scratch.allocator(); - const n: u64 = @intCast(shape_.dim(a)); - const c = (flat.origin / strides[a]) % n; - var longest: ?usize = null; - var longest_span: u64 = 0; - var total: u64 = 0; - for (flat.dims.constSlice(), 0..) |d, k| { - if (d.count <= 1 or axisOfStep(shape_.rank(), strides, d.step) != a) continue; - const span = (d.count - 1) * (d.step / strides[a]); - total += span; - if (longest == null or span > longest_span) { - longest = k; - longest_span = span; - } - } - const k = longest.?; - const d = flat.dims.get(k); - const s = d.step / strides[a]; - const rest = total - longest_span; - if (n % s == 0 and c % s + rest < s) { - const per_row = n / s; - var j: u64 = 0; - if (c >= s) { - // A partial first row. - const head = @min((n - c + s - 1) / s, d.count); - self.work.append(scratch, flat.sub(k, 0, head)) catch oom(); - j = head; - } - const rows = (d.count - j) / per_row; - if (rows > 0) { - var body = flat.sub(k, j, per_row); - body.dims.append(.{ .count = rows, .step = d.step * per_row, .stride = d.stride * per_row }) catch tooManyDims(); - self.work.append(scratch, body) catch oom(); - j += rows * per_row; - } - if (j < d.count) self.work.append(scratch, flat.sub(k, j, d.count - j)) catch oom(); - return; - } - // An irregular lattice: one piece per step of that dimension. - for (0..d.count) |step_index| { - var single = flat.sub(k, step_index, 1); - _ = single.dims.orderedRemove(k); - self.work.append(scratch, single) catch oom(); - } - } - - /// Emits in `res_shape` the elements of `axis_piece` whose coordinate on - /// `a` is in `[lo, hi)`, at that coordinate minus `lo`. - fn clip(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece, a: u3, lo: u64, hi: u64) void { - var span: u64 = 0; - var outer: ?usize = null; - for (axis_piece.dims.constSlice(), 0..) |d, k| { - if (d.axis != a or d.count <= 1) continue; - span += (d.count - 1) * d.step; - if (outer == null or d.step > axis_piece.dims.get(outer.?).step) outer = k; - } - const o = axis_piece.origin[a]; - if (o >= lo and o + span < hi) { - var kept = axis_piece; - kept.origin[a] = o - lo; - self.emit(res_shape, kept); - return; - } - if (o + span < lo or o >= hi) return; - // Cut along the outermost dimension on `a`: the steps whose whole - // sub-lattice is inside stay one piece, the partial ones recurse. - const k = outer.?; - const d = axis_piece.dims.get(k); - const inner = span - (d.count - 1) * d.step; - const full_lo: u64 = if (o >= lo) 0 else std.math.divCeil(u64, lo - o, d.step) catch unreachable; - const full_hi: u64 = if (o + inner >= hi) 0 else @min(d.count, (hi - 1 - inner - o) / d.step + 1); - if (full_lo < full_hi) { - var kept = axis_piece; - kept.dims.buffer[k].count = full_hi - full_lo; - kept.origin[a] = o + full_lo * d.step - lo; - kept.offset += full_lo * d.stride; - self.emit(res_shape, kept); - } - const touch_lo: u64 = if (o + inner >= lo) 0 else std.math.divCeil(u64, lo - o - inner, d.step) catch unreachable; - const touch_hi: u64 = @min(d.count, (hi - 1 - o) / d.step + 1); - const partial = [_][2]u64{ - .{ touch_lo, @min(full_lo, touch_hi) }, - .{ @max(full_hi, touch_lo, full_lo), touch_hi }, - }; - for (partial) |range| { - var j = range[0]; - while (j < range[1]) : (j += 1) { - var part = axis_piece; - part.origin[a] = o + j * d.step; - part.offset += j * d.stride; - _ = part.dims.orderedRemove(k); - self.clip(res_shape, part, a, lo, hi); - } - } - } - - /// Flattens `axis_piece` in `res_shape` into the result. - fn emit(self: *Rewrite, res_shape: Shape, axis_piece: AxisPiece) void { - const strides = elementStrides(res_shape); - var flat: Flat = .{ .tensor = axis_piece.tensor, .offset = axis_piece.offset, .origin = 0 }; - for (0..res_shape.rank()) |b| flat.origin += axis_piece.origin[b] * strides[b]; - for (axis_piece.dims.constSlice()) |d| { - if (d.count <= 1) continue; - flat.dims.append(.{ .count = d.count, .step = d.step * strides[d.axis], .stride = d.stride }) catch tooManyDims(); - } - self.emitFlat(flat); - } - - /// Adds `flat` to the result, its dimensions outermost first and merged - /// where one continues the other. - fn emitFlat(self: *Rewrite, flat: Flat) void { - var sorted: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; - for (flat.dims.constSlice()) |d| { - if (d.count > 1) sorted.appendAssumeCapacity(d); - } - std.mem.sort(Dim, sorted.slice(), {}, struct { - fn moreOuter(_: void, lhs: Dim, rhs: Dim) bool { - return lhs.step > rhs.step; - } - }.moreOuter); - var merged: stdx.BoundedArray(Dim, backend.max_part_dims) = .empty; - for (sorted.constSlice()) |d| { - if (merged.len != 0) { - const last = &merged.slice()[merged.len - 1]; - if (last.step == d.count * d.step and last.stride == d.count * d.stride) { - last.* = .{ .count = last.count * d.count, .step = d.step, .stride = d.stride }; - continue; - } - } - merged.appendAssumeCapacity(d); - } - const arena = self.store.arena.allocator(); - self.out.append(self.scratch.allocator(), .{ - .tensor = flat.tensor, - .offset = flat.offset, - .origin = flat.origin, - .dims = if (merged.len == 0) &.{} else arena.dupe(Dim, merged.constSlice()) catch oom(), - }) catch oom(); - } -}; - -const AxisFit = union(enum) { - piece: AxisPiece, - /// The axis some coordinate would run past. - overflow: u3, -}; - -/// `flat` in coordinates of `shape_`, if no dimension carries into the next -/// axis: each dimension moves the outermost axis its step is a multiple of. -fn fitAxes(shape_: Shape, strides: [constants.MAX_RANK]u64, flat: Flat) AxisFit { - const r = shape_.rank(); - var axis_piece: AxisPiece = .{ .tensor = flat.tensor, .offset = flat.offset }; - for (0..r) |b| axis_piece.origin[b] = (flat.origin / strides[b]) % @as(u64, @intCast(shape_.dim(b))); - var extent = axis_piece.origin; - for (flat.dims.constSlice()) |d| { - if (d.count <= 1) continue; - const b = axisOfStep(r, strides, d.step); - const step = d.step / strides[b]; - axis_piece.dims.appendAssumeCapacity(.{ .count = d.count, .axis = b, .step = step, .stride = d.stride }); - extent[b] += (d.count - 1) * step; - } - for (0..r) |b| { - if (extent[b] >= @as(u64, @intCast(shape_.dim(b)))) return .{ .overflow = @intCast(b) }; - } - return .{ .piece = axis_piece }; -} - -fn axisOfStep(r: u4, strides: [constants.MAX_RANK]u64, step: u64) u3 { - for (0..r) |b| { - if (step % strides[b] == 0) return @intCast(b); - } - std.debug.panic("a lattice step of {d} elements in a shape of rank {d}", .{ step, r }); -} - -/// Row-major element strides. -fn elementStrides(shape_: Shape) [constants.MAX_RANK]u64 { - var strides: [constants.MAX_RANK]u64 = @splat(0); - var acc: u64 = 1; - var b: usize = shape_.rank(); - while (b > 0) { - b -= 1; - strides[b] = acc; - acc *= @intCast(shape_.dim(b)); - } - return strides; -} - -fn isZero(value: anytype) bool { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_int, .int, .comptime_float, .float => value == 0, - else => false, - }; -} - -fn oom() noreturn { - @panic("OOM"); -} - -fn tooManyDims() noreturn { - std.debug.panic("a Source piece needs more than {d} dimensions", .{backend.max_part_dims}); -} - -// Tests: every op is checked against the same function run as a program -// over `zml.Tensor`, the loader's byte placement replayed on the host. - -const testing = @import("../testing.zig"); -const Buffer = @import("../buffer.zig").Buffer; - -/// A checkpoint of in-memory tensors (the registry points nowhere) and the -/// bytes each holds. -const TestCheckpoint = struct { - store: TensorStore, - bytes: std.StringHashMapUnmanaged([]u8) = .empty, - allocator: std.mem.Allocator, - - const Entry = struct { name: []const u8, shape: Shape }; - - /// Registers `entries` end to end, each holding distinct bytes. - fn init(self: *TestCheckpoint, allocator: std.mem.Allocator, entries: []const Entry) !void { - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - var offset: u64 = 0; - for (entries) |entry| { - try registry.registerTensor(.{ .file_uri = "memory", .name = entry.name, .shape = entry.shape, .offset = offset }); - offset += entry.shape.byteSize(); - } - self.* = .{ .store = .init(allocator, ®istry, .{}), .allocator = allocator }; - errdefer self.deinit(); - for (entries, 0..) |entry, index| { - const bytes = try allocator.alloc(u8, entry.shape.byteSize()); - errdefer allocator.free(bytes); - for (bytes, 0..) |*byte, i| byte.* = @truncate(i *% 7 +% index *% 61 +% 1); - try self.bytes.put(allocator, self.store.registry.tensors.getKey(entry.name).?, bytes); - } - } - - fn deinit(self: *TestCheckpoint) void { - var it = self.bytes.valueIterator(); - while (it.next()) |bytes| self.allocator.free(bytes.*); - self.bytes.deinit(self.allocator); - self.store.deinit(); - } - - /// The bytes the loader would assemble for `tensor`, placed on the host: - /// its source lowered as the loader lowers it (`Placed.lower`), the - /// bytes no part covers zero. Fails when a part writes a byte another - /// wrote, so that every byte is written exactly once, by a part or the - /// zero fill: the loader relies on disjoint pieces and does not check - /// them. - fn assemble(self: *TestCheckpoint, tensor: Tensor) ![]u8 { - var arena: std.heap.ArenaAllocator = .init(self.allocator); - defer arena.deinit(); - const parts = try self.store.bindingOf(tensor.id).?.load.source.lower(arena.allocator()); - const out = try self.allocator.alloc(u8, tensor.shape().byteSize()); - errdefer self.allocator.free(out); - @memset(out, 0); - const written = try arena.allocator().alloc(bool, out.len); - @memset(written, false); - for (parts) |part| { - const src = self.bytes.get(part.source.name).?; - const width: usize = @intCast(part.lattice.width); - var runs: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); - while (runs.next()) |run| { - const at: usize = @intCast(run.dst); - for (written[at..][0..width]) |*byte| { - if (byte.*) return error.DoubleWrite; - byte.* = true; - } - @memcpy(out[at..][0..width], src[@intCast(run.src)..][0..width]); - } - } - return out; - } -}; - -fn interleave(comptime T: type, members: []const T, groups: i64) T { - var parts: [4]T = undefined; - for (members, parts[0..members.len]) |member, *part| part.* = member.splitAxis(.row, .{ .group = groups, .r = -1 }); - return T.concatenate(parts[0..members.len], .r).merge(.{ .row = .{ .group, .r } }); -} - -const qkv_inputs = .{ .{ .row = 4, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; -const expert_inputs = .{ .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 }, .{ .row = 2, .col = 3 } }; - -/// `build(T, x)` builds a tensor from `x` with shape ops only, the same code -/// for `Tensor` and `Source`; `inputs` holds the dims of each `u16` input. -fn expectCase(comptime inputs: anytype, comptime build: anytype) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - const n = inputs.len; - var names: [n][]const u8 = undefined; - var name_buffers: [n][16]u8 = undefined; - var shapes: [n]Shape = undefined; - var entries: [n]TestCheckpoint.Entry = undefined; - inline for (inputs, 0..) |dims_, i| { - names[i] = try std.fmt.bufPrint(&name_buffers[i], "input{d}", .{i}); - shapes[i] = Shape.init(dims_, .u16); - entries[i] = .{ .name = names[i], .shape = shapes[i] }; - } - var checkpoint: TestCheckpoint = undefined; - try checkpoint.init(allocator, &entries); - defer checkpoint.deinit(); - - var sources: [n]Source = undefined; - for (&sources, names) |*s, input_name| s.* = checkpoint.store.view().source(input_name, null); - const bound = build(Source, &sources).withPartitioning(.replicated, .{}).bind(); - const assembled = try checkpoint.assemble(bound); - defer allocator.free(assembled); - - const platform = testing.env(); - var tensors: [n]Tensor = undefined; - for (&tensors, shapes) |*t, shape_| t.* = .fromShape(shape_); - const Program = struct { - fn forward(x: []const Tensor) Tensor { - return build(Tensor, x); - } - }; - var exe = try platform.compileFn(allocator, io, Program.forward, .{&tensors}, .{}); - defer exe.deinit(); - var buffers: [n]Buffer = undefined; - var created: usize = 0; - defer for (buffers[0..created]) |*buffer| buffer.deinit(); - for (&buffers, shapes, names) |*buffer, shape_, input_name| { - buffer.* = try .fromBytes(io, platform, shape_, checkpoint.bytes.get(input_name).?); - created += 1; - } - var args = try exe.args(allocator); - defer args.deinit(allocator); - var results = try exe.results(allocator); - defer results.deinit(allocator); - args.set(.{buffers[0..]}); - try exe.call(io, args, &results, .{ .wait = true }); - var output = results.get(Buffer); - defer output.deinit(); - const expected = try output.toSliceAlloc(allocator, io); - defer expected.free(allocator); - - std.testing.expect(bound.shape().eqlWithTags(output.shape())) catch |err| { - std.log.err("Source gives {f}, Tensor {f}", .{ bound.shape(), output.shape() }); - return err; - }; - std.testing.expectEqualSlices(u8, expected.constData(), assembled) catch |err| { - std.log.err("the loader's bytes differ from the program's", .{}); - return err; - }; -} - -test "Source concatenates rows as Tensor does" { - try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 2, .col = 4 }, .{ .row = 4, .col = 4 } }, struct { - fn build(comptime T: type, x: []const T) T { - return T.concatenate(x, .row); - } - }.build); -} - -test "Source concatenates columns as Tensor does" { - try expectCase(.{ .{ .row = 3, .col = 4 }, .{ .row = 3, .col = 2 } }, struct { - fn build(comptime T: type, x: []const T) T { - return T.concatenate(x, .col); - } - }.build); -} - -test "Source interleaves q, k and v by heads as Tensor does" { - try expectCase(qkv_inputs, struct { - fn build(comptime T: type, x: []const T) T { - return interleave(T, x, 2); - } - }.build); -} - -test "Source interleaves rows as Tensor does" { - try expectCase(.{ .{ .row = 4, .col = 3 }, .{ .row = 4, .col = 3 } }, struct { - fn build(comptime T: type, x: []const T) T { - return interleave(T, x, 4); - } - }.build); -} - -test "Source stacks experts as Tensor does" { - try expectCase(expert_inputs, struct { - fn build(comptime T: type, x: []const T) T { - const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; - return T.concatenate(&members, .row); - } - }.build); -} - -test "Source interleaves stacked experts as Tensor does" { - try expectCase(expert_inputs, struct { - fn build(comptime T: type, x: []const T) T { - const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; - return T.stack(&members, 2, .proj).merge(.{ .rows = .{ .row, .proj } }); - } - }.build); -} - -test "Source interleaves expert vectors as Tensor does" { - // Per-expert vectors (biases): row r of projection p lands at r * 2 + p. - try expectCase(.{ .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 }, .{ .row = 3 } }, struct { - fn build(comptime T: type, x: []const T) T { - const members = [2]T{ T.stack(x[0..2], 0, .expert), T.stack(x[2..4], 0, .expert) }; - return T.stack(&members, .last, .proj).merge(.{ .rows = .{ .row, .proj } }); - } - }.build); -} - -test "Source transposes as Tensor does" { - try expectCase(.{.{ .row = 3, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].transpose(.{ .col, .row }); - } - }.build); -} - -test "Source interleaves rows of three as Tensor does" { - // lfm2's in_proj: rows [B; C; x] become b0 c0 x0 b1 c1 x1. - try expectCase(.{.{ .row = 6, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].splitAxis(.row, .{ .m = 3, .c = 2 }).transpose(.{ .c, .m, .col }).merge(.{ .row = .{ .c, .m } }); - } - }.build); -} - -test "Source slices rows as Tensor does" { - try expectCase(.{.{ .row = 5, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].slice(.row, .{ .start = 1, .end = 4 }); - } - }.build); -} - -test "Source slices columns as Tensor does" { - try expectCase(.{.{ .row = 3, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].slice(.col, .{ .start = 1, .end = 3 }); - } - }.build); -} - -test "Source slices by length as Tensor does" { - try expectCase(.{.{ .row = 6, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].slice(.row, .{ .start = 2, .len = 3 }); - } - }.build); -} - -test "Source slices an interleave as Tensor does" { - try expectCase(qkv_inputs, struct { - fn build(comptime T: type, x: []const T) T { - return interleave(T, x, 2).slice(.row, .{ .start = 3, .end = 7 }); - } - }.build); -} - -test "Source takes a single row as Tensor does" { - try expectCase(.{.{ .row = 5, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].slice(.row, .single(2)); - } - }.build); -} - -test "Source pads as Tensor does" { - try expectCase(.{.{ .row = 4, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 4 }, .col = Tensor.Pad{ .low = 1, .interior = 1 } }); - } - }.build); -} - -test "Source pads rows as Tensor does" { - try expectCase(.{.{ .row = 5, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].pad(0, .{ .row = Tensor.Pad{ .high = 3 } }); - } - }.build); -} - -test "Source broadcasts a scalar as Tensor does" { - try expectCase(.{.{}}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].broad(Shape.init(.{ .row = 3, .col = 2 }, .u16)); - } - }.build); -} - -test "Source broadcasts a column as Tensor does" { - // A per-tensor scale repeated over each member's rows. - try expectCase(.{ .{ .row = 1, .col = 1 }, .{ .row = 1, .col = 1 } }, struct { - fn build(comptime T: type, x: []const T) T { - const members = [2]T{ x[0].broad(Shape.init(.{ .row = 4, .col = 1 }, .u16)), x[1].broad(Shape.init(.{ .row = 2, .col = 1 }, .u16)) }; - return interleave(T, &members, 2); - } - }.build); -} - -test "Source bitcasts to a narrower type as Tensor does" { - try expectCase(.{.{ .row = 3, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].bitCast(.u8).merge(.{ .col = .{ .col, .bitcast } }); - } - }.build); -} - -test "Source bitcasts to a wider type as Tensor does" { - try expectCase(.{.{ .row = 2, .col = 6 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].bitCast(.u8).splitAxis(.col, .{ .col = 3, .pair = 2 }).merge(.{ .pair = .{ .pair, .bitcast } }).bitCast(.u32); - } - }.build); -} - -test "Source swizzles block scales as Tensor does" { - // The 128x4 block-scale layout, at 8x4: rows padded to 8 and tiled - // by 2x4, columns padded to 4. - try expectCase(.{.{ .e = 2, .m = 7, .k = 6 }}, struct { - fn build(comptime T: type, x: []const T) T { - return x[0].pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) - .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) - .splitAxis(.k, .{ .ko = -1, .ki = 4 }) - .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) - .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); - } - }.build); -} - -test "Source splits then interleaves as Tensor does" { - // qwen3_5's in_proj_qkv sliced into q, k and v, interleaved with z. - try expectCase(.{ .{ .row = 6, .col = 3 }, .{ .row = 2, .col = 3 } }, struct { - fn build(comptime T: type, x: []const T) T { - const qkv = x[0]; - const members = [4]T{ - qkv.slice(.row, .{ .start = 0, .end = 2 }), - qkv.slice(.row, .{ .start = 2, .end = 4 }), - qkv.slice(.row, .{ .start = 4, .end = 6 }), - x[1], - }; - return interleave(T, &members, 2); - } - }.build); -} - -test "Source chunks then stacks as Tensor does" { - try expectCase(.{.{ .row = 6, .col = 4 }}, struct { - fn build(comptime T: type, x: []const T) T { - const chunks = x[0].chunkExact(.row, 3); - return T.stack(&.{ chunks[2], chunks[0] }, .last, .pair).insertAxes(0, .{.one}).squeeze(0); - } - }.build); -} - -test "a source binds as its pieces, which the loader lowers to byte parts" { - var checkpoint: TestCheckpoint = undefined; - try checkpoint.init(std.testing.allocator, &.{ - .{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }, - .{ .name = "k", .shape = .init(.{ 2, 3 }, .u16) }, - .{ .name = "v", .shape = .init(.{ 2, 3 }, .u16) }, - }); - defer checkpoint.deinit(); - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const view = checkpoint.store.view(); - const q = view.source("q", .{ .row, .col }); - const entries = [_]*const safetensors.Tensor{ view.get("q").?, view.get("k").?, view.get("v").? }; - - // Relabels keep the whole tensor, one run of it; only `bind` records - // anything, and the binding keeps the placed source as it is. - const relabeled = q.splitAxis(.row, .{ .h = 2, .hd = 2 }).merge(.{ .row = .{ .h, .hd } }).withPartitioning(.replicated, .{}); - try std.testing.expectEqual(@as(usize, 0), checkpoint.store.id_to_sources.count()); - const bound = checkpoint.store.bindingOf(relabeled.bind().id).?.load.source; - try std.testing.expectEqual(relabeled._pieces, bound._pieces); - try std.testing.expectEqual(entries[0], bound.whole().?); - try std.testing.expectEqualDeep(&[_]backend.LoadPart{.whole(entries[0])}, try bound.lower(arena.allocator())); - // One checkpoint tensor read twice is two pieces of it. - const twice = Source.concatenate(&.{ q, view.source("q", .{ .row, .col }) }, .row).withPartitioning(.replicated, .{}); - try std.testing.expectEqual(null, twice.whole()); - try std.testing.expectEqualDeep(&[_]backend.LoadPart{ - .{ .source = entries[0], .lattice = .{ .dst_offset = 0, .width = 24 } }, - .{ .source = entries[0], .lattice = .{ .dst_offset = 24, .width = 24 } }, - }, try twice.lower(arena.allocator())); - - // q, k and v in two groups: each member is one part of two runs, its - // rows of a group side by side with the others'. - const members = [_]Source{ q, view.source("k", .{ .row, .col }), view.source("v", .{ .row, .col }) }; - const fused = interleave(Source, &members, 2).withPartitioning(.replicated, .{}); - try std.testing.expectEqualDeep(&[_]backend.LoadPart{ - .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 12, .dims = &.{.{ .count = 2, .src_stride = 12, .dst_stride = 24 }} } }, - .{ .source = entries[1], .lattice = .{ .src_offset = 0, .dst_offset = 12, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, - .{ .source = entries[2], .lattice = .{ .src_offset = 0, .dst_offset = 18, .width = 6, .dims = &.{.{ .count = 2, .src_stride = 6, .dst_stride = 24 }} } }, - }, try fused.lower(arena.allocator())); - - // A transpose is one part of single elements. - const transposed = q.transpose(.{}).withPartitioning(.replicated, .{}); - try std.testing.expectEqualDeep(&[_]backend.LoadPart{ - .{ .source = entries[0], .lattice = .{ .src_offset = 0, .dst_offset = 0, .width = 2, .dims = &.{ - .{ .count = 3, .src_stride = 2, .dst_stride = 8 }, - .{ .count = 4, .src_stride = 6, .dst_stride = 2 }, - } } }, - }, try transposed.lower(arena.allocator())); -} - -test "assembling fails on a byte two pieces write" { - var checkpoint: TestCheckpoint = undefined; - try checkpoint.init(std.testing.allocator, &.{.{ .name = "q", .shape = .init(.{ 4, 3 }, .u16) }}); - defer checkpoint.deinit(); - // Only a hand-built binding can repeat a piece. - const q = checkpoint.store.view().source("q", .{ .row, .col }); - const pieces = try checkpoint.store.arena.allocator().dupe(Piece, &.{ q._pieces[0], q._pieces[0] }); - const tensor = bindNew(&checkpoint.store, q.shape(), .{ .load = .{ .source = .{ .store = &checkpoint.store, .shape = q.shape(), ._pieces = pieces } } }); - try std.testing.expectError(error.DoubleWrite, checkpoint.assemble(tensor)); -} - -test "a source is placed last, by as or withPartitioning, and binds in its placed shape" { - const allocator = std.testing.allocator; - var checkpoint: TestCheckpoint = undefined; - try checkpoint.init(allocator, &.{.{ .name = "w", .shape = .init(.{ 4, 6 }, .u16) }}); - defer checkpoint.deinit(); - const mesh: Sharding.Mesh = .initTest(.tp, .{.model}); - const transposed = checkpoint.store.view().source("w", .{ .row, .col }).transpose(.{}); - - // `as` takes a shape built elsewhere, here of another rank: its - // dimensions, tags and sharding, for the same elements. - const placed = Shape.init(.{ .flat = 24 }, .u16).withPartitioning(&mesh, .{ .flat = .model }); - const placed_source = transposed.as(placed); - const flat = placed_source.bind(); - try std.testing.expect(flat.shape().eqlWithTagsAndPartitioning(placed)); - try std.testing.expect(checkpoint.store.bindingOf(flat.id).?.load.source.shape.eqlWithTagsAndPartitioning(placed)); - const replicated = transposed.withPartitioning(.replicated, .{}).bind(); - try std.testing.expect(replicated.shape().eqlWithTags(transposed.shape())); - try std.testing.expectEqual(null, replicated.shape()._sharding.mesh); - // `unplaced` keeps the placed dimensions and tags, without the sharding. - const dropped = placed_source.unplaced(); - try std.testing.expect(dropped.shape().eqlWithTags(placed)); - try std.testing.expectEqual(null, dropped.shape()._sharding.mesh); - - const flat_bytes = try checkpoint.assemble(flat); - defer allocator.free(flat_bytes); - const replicated_bytes = try checkpoint.assemble(replicated); - defer allocator.free(replicated_bytes); - try std.testing.expectEqualSlices(u8, replicated_bytes, flat_bytes); -} - -test "lattices stay one piece per source at checkpoint sizes" { - // Registered without bytes: nothing is evaluated. - var registry: safetensors.TensorRegistry = .init(std.testing.allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .file_uri = "memory", .name = "gate", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); - try registry.registerTensor(.{ .file_uri = "memory", .name = "up", .shape = .init(.{ 14336, 4096 }, .bf16), .offset = 0 }); - var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); - defer store.deinit(); - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const view = store.view(); - const members = [_]Source{ view.source("gate", .{ .row, .col }), view.source("up", .{ .row, .col }) }; - - // One row per group: two pieces, each 14336 runs of one row. - const finest = interleave(Source, &members, 14336); - try std.testing.expectEqual(@as(usize, 2), finest._pieces.len); - for (try finest.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { - try std.testing.expectEqual(@as(u64, 14336), part.lattice.runs()); - try std.testing.expectEqual(@as(u64, 8192), part.lattice.width); - } - // One group per device of four: four runs of 28 MiB each. - const per_shard = interleave(Source, &members, 4); - for (try per_shard.withPartitioning(.replicated, .{}).lower(arena.allocator())) |part| { - try std.testing.expectEqual(@as(u64, 4), part.lattice.runs()); - try std.testing.expectEqual(@as(u64, 3584 * 8192), part.lattice.width); - } - // Slicing through the interleave cuts at most a few pieces. - const sliced = finest.slice(.row, .{ .start = 1001, .end = 20001 }); - try std.testing.expect(sliced._pieces.len <= 6); -} diff --git a/zml/io/TensorStore.zig b/zml/io/TensorStore.zig deleted file mode 100644 index 3b159bbc..00000000 --- a/zml/io/TensorStore.zig +++ /dev/null @@ -1,558 +0,0 @@ -//! Associates model tensors with checkpoint sources and prefixed views, -//! over the entries it takes from a checkpoint registry. -//! -//! The store owns those entries, the bindings, and every source and program -//! value (its arena). Views and sources point to it: once `view` was -//! called it must not move, and it must outlive them and every load -//! started from it (`Loader.start` borrows it until `wait` or `cancel`). -//! It borrows the meshes of `Options.meshes`, which belong to the platform. -//! Binding cannot fail: a missing key panics with the key, and what the -//! loader cannot fill is its error (`Loader.start`'s `InvalidBinding`). -const std = @import("std"); - -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const Exe = @import("../exe.zig").Exe; -const MemorySpace = @import("../platform.zig").Memory.Kind; -const Platform = @import("../platform.zig").Platform; -const Quantization = @import("../quantization.zig").Quantization; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; -const load_log = @import("log.zig").load; -const testing = @import("../testing.zig"); - -const TensorStore = @This(); - -/// Where the loader delivers a loaded tensor, on every device of its mesh -/// (`View.setMemory`). The destination is allocated once per device -/// whatever its pieces; the host kinds take no device memory. -pub const Memory = enum { - /// Device memory. - default, - /// Buffers in each device's pinned host memory space. An executable - /// that takes them as device inputs copies them to the device first. - host_pinned, - /// Pinned host memory that executables read in place: each device's - /// buffer is a device-kind view of it (`createViewOfDeviceBuffer`), - /// and the memory goes with the last view. For a table too large for - /// device memory, of which a step reads a few rows. XLA counts the - /// views against device memory (`View.hostMappedBytes`). - host_mapped, - - /// Whether the bytes live in host memory, written by the CPU. - pub fn isHost(self: Memory) bool { - return switch (self) { - .default => false, - .host_pinned, .host_mapped => true, - }; - } - - /// The memory space of each device's delivered buffer. - pub fn space(self: Memory) MemorySpace { - return switch (self) { - .default, .host_mapped => .default, - .host_pinned => .host_pinned, - }; - } -}; - -/// What the loader reads of a bound tensor (`bindingOf`): the source -/// it fills the tensor from, or the program that computes it. -pub const Binding = union(enum) { - /// The loader fills the tensor from `source` (`Source.Placed.bind`), - /// every byte no piece covers zero. - load: Load, - /// A program computes the tensor while loading (`Source.bindProgram`). - /// Its output lands in default device memory. - compute: Compute, - - pub const Load = struct { - source: Source.Placed, - /// Where the loader delivers this tensor (`View.setMemory`). - memory: Memory = .default, - }; - - pub const Compute = struct { - program: Program, - /// The program's inputs in order, each landed in default device - /// memory, placed by its shape. - inputs: []const Source.Placed, - - /// Whether this computing `output` and `other` computing - /// `other_output` compile to the same executable: equal programs - /// over the same shapes placed alike. - pub fn sameExecutable(self: Compute, output: Shape, other: Compute, other_output: Shape) bool { - if (!self.program.eql(other.program)) return false; - if (!output.eqlWithTagsAndPartitioning(other_output) or self.inputs.len != other.inputs.len) return false; - for (self.inputs, other.inputs) |a, b| { - if (!a.shape.eqlWithTagsAndPartitioning(b.shape)) return false; - } - return true; - } - - /// Compiles the program computing a tensor of `output` from inputs - /// of the shapes it binds, its module named after the program. - pub fn compile( - self: Compute, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - meshes: []const *const Sharding.Mesh, - output: Shape, - ) Compiler.Error!Exe { - const inputs = try allocator.alloc(Tensor, self.inputs.len); - defer allocator.free(inputs); - for (inputs, self.inputs) |*input, source| input.* = .fromShape(source.shape); - const opts: Compiler.Options = .{ .meshes = meshes, .program_name = self.program.name() }; - return self.program.vtable.compile(self.program.value, allocator, io, platform, opts, output, inputs); - } - }; -}; - -/// A program the loader compiles and runs to compute a tensor while loading -/// (`Binding.compute`): `forward(output, inputs)` of the program's value. -/// Model code makes one through `Source.bindProgram`, and reads it back -/// with `as`. -pub const Program = struct { - /// The program's value, in the store's arena. - value: *const anyopaque, - vtable: *const VTable, - - const VTable = struct { - name: []const u8, - compile: *const fn ( - value: *const anyopaque, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Compiler.Options, - output: Shape, - inputs: []const Tensor, - ) Compiler.Error!Exe, - eql: *const fn (a: *const anyopaque, b: *const anyopaque) bool, - }; - - /// `program`, its value copied into `arena`. Its type declares - /// `pub fn forward(self, output: Shape, inputs: []const Tensor) Tensor` - /// and is plain data, without pointers, so equal values are found by - /// value and compile once. - pub fn init(arena: std.mem.Allocator, program: anytype) std.mem.Allocator.Error!Program { - const T = @TypeOf(program); - comptime { - if (!isPlainData(T)) @compileError("TensorStore.Program: " ++ @typeName(T) ++ " must be plain data, without pointers"); - } - const copy = try arena.create(T); - copy.* = program; - return .{ .value = copy, .vtable = &Erased(T).vtable }; - } - - /// The program's value, if it is a `T`. - pub fn as(self: Program, comptime T: type) ?T { - if (self.vtable != &Erased(T).vtable) return null; - return Erased(T).typed(self.value); - } - - /// The program's type name, which names its compiled module in traces, - /// dumps and logs. - pub fn name(self: Program) []const u8 { - return self.vtable.name; - } - - /// Whether `a` and `b` are programs of one type with equal values. - fn eql(a: Program, b: Program) bool { - return a.vtable == b.vtable and a.vtable.eql(a.value, b.value); - } - - /// The vtable of programs of type `T`, one per type. - fn Erased(comptime T: type) type { - return struct { - const vtable: VTable = .{ .name = @typeName(T), .compile = compileValue, .eql = eqlValues }; - - fn typed(erased: *const anyopaque) T { - return @as(*const T, @ptrCast(@alignCast(erased))).*; - } - - fn compileValue( - erased: *const anyopaque, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Compiler.Options, - output: Shape, - inputs: []const Tensor, - ) Compiler.Error!Exe { - return Compiler.compile(allocator, io, platform, T.forward, .{ typed(erased), output, inputs }, opts); - } - - fn eqlValues(a: *const anyopaque, b: *const anyopaque) bool { - return std.meta.eql(typed(a), typed(b)); - } - }; - } - - /// Values compared by value: no pointer anywhere inside. - fn isPlainData(comptime T: type) bool { - return switch (@typeInfo(T)) { - .void, .bool, .int, .float, .@"enum" => true, - .array => |info| isPlainData(info.child), - .optional => |info| isPlainData(info.child), - .@"struct" => |info| inline for (info.field_types) |Field| { - if (!isPlainData(Field)) break false; - } else true, - .@"union" => |info| info.tag_type != null and inline for (info.field_types) |Field| { - if (!isPlainData(Field)) break false; - } else true, - else => false, - }; - } -}; - -pub const Source = @import("Source.zig"); - -registry: safetensors.TensorRegistry, -meshes: []const *const Sharding.Mesh, -quantization: ?Quantization.Target, -id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), -allocator: std.mem.Allocator, -arena: std.heap.ArenaAllocator, - -pub const Options = struct { - /// The meshes `Source.withPartitioning` and `View.mesh` name. The slice - /// is copied; the meshes must outlive the store. - meshes: []const *const Sharding.Mesh = &.{}, - /// How the platform's FP8 GEMMs want block-FP8 weights, for the code - /// that binds them (`View.quantization`): `.of(platform)`. Required to - /// bind block-FP8 weights. - quantization: ?Quantization.Target = null, -}; - -/// A store over the entries of `registry`, which must be complete: add -/// every checkpoint first (`TensorRegistry.addRepo`). It takes the entries -/// and leaves `registry` empty. The store's map stays locked until -/// `deinit`, so a later insert panics in safe builds, and an entry's -/// address is its checkpoint tensor's identity (`View.get`). -pub fn init(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, opts: Options) TensorStore { - var arena: std.heap.ArenaAllocator = .init(allocator); - const meshes = arena.allocator().dupe(*const Sharding.Mesh, opts.meshes) catch @panic("OOM"); - var self: TensorStore = .{ - .registry = registry.*, - .meshes = meshes, - .quantization = opts.quantization, - .id_to_sources = .empty, - .allocator = allocator, - .arena = arena, - }; - registry.* = .init(registry.arena.child_allocator); - self.registry.tensors.lockPointers(); - return self; -} - -/// Frees the bindings, the sources and programs, then the registry entries -/// `init` took. No load started from the store may still run: `wait` or -/// `cancel` it first. -pub fn deinit(self: *TensorStore) void { - self.id_to_sources.deinit(self.allocator); - self.arena.deinit(); - self.registry.tensors.unlockPointers(); - self.registry.deinit(); -} - -/// The binding of tensor `id`, as the loader reads it; null when this store -/// did not bind it. -pub fn bindingOf(self: *const TensorStore, id: Tensor.Id) ?Binding { - return self.id_to_sources.get(id); -} - -/// The root view, with no prefix. It points to this store, which must not -/// move afterwards. -pub fn view(self: *TensorStore) View { - return .{ .store = self }; -} - -/// The mesh `name` among `Options.meshes`; model code asks its `View`. -pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (store.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - std.debug.panic( - \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.io.TensorStore.init`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(store.meshes) }); -} - -/// What model code holds: a key prefix over the store's checkpoint, to look -/// tensors up, bind them and set their memory. A value that keeps its own -/// copy of the prefix (256 bytes at most, longer panics) and points to the -/// store. -pub const View = struct { - store: *TensorStore, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - /// The store's root view, with no prefix. - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - /// This view under `prefix_` and a dot: `withPrefix("model")` then - /// `get("embed")` looks up `model.embed`. It only scopes lookups; - /// nothing is bound. - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix(), prefix_ }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - /// This view under `index` and a dot, as `withPrefix` of the number. - pub fn withLayer(self: *const View, index: usize) View { - var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix(), index }); - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - /// The keys' prefix under this view, with its trailing dot; "" at the - /// root. - pub fn prefix(self: *const View) []const u8 { - return self.prefix_buffer[0..self.prefix_length]; - } - - /// The checkpoint tensor `subkey`, the registry's own entry, whose - /// address is the tensor's identity. Null when the checkpoint has none. - pub fn get(self: View, subkey: []const u8) ?*const safetensors.Tensor { - var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix(), subkey }); - return self.store.registry.tensors.getPtr(key); - } - - /// The checkpoint tensor `subkey`, with `tags` (null keeps the - /// checkpoint's), placed on `mesh_` by `partitioning` and bound: - /// `source(subkey, tags).withPartitioning(mesh_, partitioning).bind()`. - /// Panics when the checkpoint has no such tensor. - pub fn createTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); - } - - /// Like `createTensor`, but null when the checkpoint has no such tensor. - pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, comptime mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const source_ = self.maybeSource(subkey, tags) orelse return null; - return source_.withPartitioning(mesh_, partitioning).bind(); - } - - /// `createTensor` replicated on every device, without a mesh. - pub fn createReplicatedTensor(self: View, subkey: []const u8, comptime tags: anytype) Tensor { - return self.createTensor(subkey, tags, .replicated, .{}); - } - - /// Changes where the loader delivers `tensor` (`Memory`), for a - /// decision made after the tensor was bound. `tensor` must be bound in - /// this store, and not computed by a program: a program's output lands - /// in default device memory. - pub fn setMemory(self: View, tensor: Tensor, kind: Memory) void { - const binding = self.store.id_to_sources.getPtr(tensor.id) orelse - stdx.debug.panic("Tensor {} {f} has no binding in this store", .{ tensor.id, tensor.shape() }); - switch (binding.*) { - .load => |*load| load.memory = kind, - .compute => |compute| stdx.debug.panic("setMemory: program {s} computes tensor {} {f}, whose output lands in default device memory", .{ compute.program.name(), tensor.id, tensor.shape() }), - } - } - - /// The bytes on one device of every tensor the store delivers - /// `.host_mapped`, each counted by its shard: what XLA counts against - /// device memory for those views, which a model gives back through - /// `Compiler.Options.extra_device_memory_bytes`. The whole store's, - /// whatever this view's prefix; ask once every binding and `setMemory` - /// is done. A tensor whose mesh partitions do not divide its dimensions - /// is `IncompatibleSharding`, logged. - pub fn hostMappedBytes(self: View) !u64 { - var bytes: u64 = 0; - var it = self.store.id_to_sources.iterator(); - while (it.next()) |entry| switch (entry.value_ptr.*) { - .load => |load| if (load.memory == .host_mapped) { - const shard = Sharding.shardedShape(load.source.shape) catch |err| { - load_log.err("tensor {} {f}, delivered host_mapped: its dimensions are not divisible by its mesh partitions", .{ entry.key_ptr.*, load.source.shape }); - return err; - }; - bytes += shard.byteSize(); - }, - .compute => {}, - }; - return bytes; - } - - /// See `TensorStore.mesh`. - pub fn mesh(self: View, name: @EnumLiteral()) *const Sharding.Mesh { - return self.store.mesh(name); - } - - /// See `Options.quantization`. Panics when the store has none: binding - /// block-FP8 weights requires it. - pub fn quantization(self: View) Quantization.Target { - return self.store.quantization orelse - stdx.debug.panic("TensorStore.Options.quantization is unset: block-FP8 weights bind for a platform; pass .of(platform)", .{}); - } - - /// The checkpoint tensor `subkey` as a `Source`, with `tags` (null keeps - /// the checkpoint's). Panics when the checkpoint has no such tensor. - pub fn source(self: View, subkey: []const u8, comptime tags: anytype) Source { - return self.maybeSource(subkey, tags) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix(), subkey }); - } - - /// Like `source`, but null when the checkpoint has no such tensor. - pub fn maybeSource(self: View, subkey: []const u8, comptime tags: anytype) ?Source { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - return .init(self, subkey, if (has_tags) parsed_tags.constSlice() else null); - } - - /// The checkpoint tensors under this view's prefix: a scan of every key. - pub fn count(self: View) usize { - var count_: usize = 0; - const prefix_ = self.prefix(); - var it = self.store.registry.tensors.iterator(); - while (it.next()) |item| { - const key = item.key_ptr.*; - if (std.mem.startsWith(u8, key, prefix_)) { - count_ += 1; - } - } - return count_; - } - - fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { - const key = std.fmt.bufPrint(buffer, fmt, args) catch - std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); - return key; - } -}; - -test "get finds exact keys, the registry's own entries, and count the keys under a prefix" { - var registry: safetensors.TensorRegistry = .init(std.testing.allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .file_uri = "weights", .name = "layer.experts.0.down_proj.weight", .shape = .init(.{4}, .u8), .offset = 0 }); - var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); - defer store.deinit(); - // The store took the entries. - try std.testing.expectEqual(0, registry.tensors.count()); - - const layer = store.view().withPrefix("layer"); - const entry = layer.get("experts.0.down_proj.weight").?; - try std.testing.expectEqual(entry, store.registry.tensors.getPtr("layer.experts.0.down_proj.weight").?); - try std.testing.expectEqual(null, layer.get("experts.0.down_proj")); - try std.testing.expectEqual(null, layer.get("weights")); - try std.testing.expectEqualStrings("layer.", layer.prefix()); - try std.testing.expectEqualStrings("", layer.root().prefix()); - try std.testing.expectEqualStrings("layer.experts.0.", layer.withPrefix("experts").withLayer(0).prefix()); - try std.testing.expectEqual(1, layer.withPrefix("experts").count()); - try std.testing.expectEqual(0, layer.withPrefix("gate").count()); - - // Every binding of the entry holds the entry itself. - const tensor = layer.createReplicatedTensor("experts.0.down_proj.weight", null); - try std.testing.expectEqual(entry, store.bindingOf(tensor.id).?.load.source.whole().?); -} - -test "programs of one type and equal values over the same placed shapes share an executable" { - var registry: safetensors.TensorRegistry = .init(std.testing.allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .file_uri = "weights", .name = "t", .shape = .init(.{4}, .u8), .offset = 0 }); - var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); - defer store.deinit(); - const Scale = struct { - factor: u8, - - pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { - return inputs[0]; - } - }; - const Shift = struct { - factor: u8, - - pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { - return inputs[0]; - } - }; - const shape: Shape = .init(.{4}, .u8); - const input = store.view().source("t", null).withPartitioning(.replicated, .{}); - const tensors = [_]Tensor{ - Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), - Source.bindProgram(&.{input}, shape, Scale{ .factor = 1 }), - Source.bindProgram(&.{input}, shape, Scale{ .factor = 2 }), - Source.bindProgram(&.{input}, shape, Shift{ .factor = 1 }), - }; - var computes: [tensors.len]Binding.Compute = undefined; - for (&computes, tensors) |*compute, tensor| compute.* = store.bindingOf(tensor.id).?.compute; - try std.testing.expect(computes[0].sameExecutable(shape, computes[1], shape)); - try std.testing.expect(!computes[0].sameExecutable(shape, computes[2], shape)); - try std.testing.expect(!computes[0].sameExecutable(shape, computes[3], shape)); - try std.testing.expect(!computes[0].sameExecutable(shape, computes[1], shape.withDtype(.i8))); - try std.testing.expectEqual(1, computes[3].program.as(Shift).?.factor); - try std.testing.expect(computes[3].program.as(Scale) == null); -} - -test "setMemory changes where the loader delivers a bound tensor" { - var registry: safetensors.TensorRegistry = .init(std.testing.allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .file_uri = "weights", .name = "table", .shape = .init(.{ 4, 2 }, .u8), .offset = 0 }); - var store: TensorStore = .init(std.testing.allocator, ®istry, .{}); - defer store.deinit(); - - const view_ = store.view(); - const tensor = view_.createReplicatedTensor("table", null); - try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); - view_.setMemory(tensor, .host_mapped); - try std.testing.expectEqual(Memory.host_mapped, store.bindingOf(tensor.id).?.load.memory); - view_.setMemory(tensor, .default); - try std.testing.expectEqual(Memory.default, store.bindingOf(tensor.id).?.load.memory); -} - -test "hostMappedBytes counts one device's shard of each host_mapped tensor" { - const allocator = std.testing.allocator; - var devices: testing.Devices = undefined; - try devices.init(allocator, std.testing.io, 2); - defer devices.deinit(); - const tp = try devices.mesh("tp"); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .file_uri = "weights", .name = "rows", .shape = .init(.{ 8, 2 }, .u8), .offset = 0 }); - try registry.registerTensor(.{ .file_uri = "weights", .name = "scale", .shape = .init(.{4}, .u8), .offset = 16 }); - try registry.registerTensor(.{ .file_uri = "weights", .name = "dense", .shape = .init(.{4}, .u8), .offset = 20 }); - try registry.registerTensor(.{ .file_uri = "weights", .name = "odd", .shape = .init(.{ 3, 2 }, .u8), .offset = 24 }); - var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{&tp} }); - defer store.deinit(); - - const view_ = store.view(); - const rows = view_.createTensor("rows", .{ .row, .col }, .tp, .{ .row = .model }); - const scale = view_.createReplicatedTensor("scale", null); - _ = view_.createReplicatedTensor("dense", null); - try std.testing.expectEqual(0, try view_.hostMappedBytes()); - view_.setMemory(rows, .host_mapped); - view_.setMemory(scale, .host_mapped); - // Half the rows, and the whole replicated scale, from any prefix. - try std.testing.expectEqual(8 + 4, try view_.withPrefix("rows").hostMappedBytes()); - - // Three rows do not split over two devices. - const odd = view_.createTensor("odd", .{ .row, .col }, .tp, .{ .row = .model }); - view_.setMemory(odd, .host_mapped); - try std.testing.expectError(error.IncompatibleSharding, view_.hostMappedBytes()); -} diff --git a/zml/io/backend.zig b/zml/io/backend.zig deleted file mode 100644 index f118f5fe..00000000 --- a/zml/io/backend.zig +++ /dev/null @@ -1,625 +0,0 @@ -//! The seam between the front end (loader.zig) and its backends: the -//! `LoadSpec` contract they trust, its parts' byte runs (`Lattice`), the -//! source request size (`requestSize`), and which backend and memory kinds -//! a platform takes (`loadsDirect`, `supportsMemory`). -const std = @import("std"); - -const VFS = @import("vfs"); - -const Buffer = @import("../buffer.zig").Buffer; -const Platform = @import("../platform.zig").Platform; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const Memory = @import("TensorStore.zig").Memory; -const constants = @import("../constants.zig"); -const buffered_loader = @import("buffered_loader.zig"); -const direct_loader = @import("direct_loader.zig"); -const dma_calibration = @import("dma_calibration.zig"); - -/// The largest source request, and so the largest DMA block: a request is -/// a whole number of blocks, at least one. -pub const max_read_request_size: usize = 32 * 1024 * 1024; - -/// The source request for DMA blocks of `block_size`: the read chunk -/// rounded up to whole blocks, one block at least. Null unless the block is -/// a nonzero multiple of `alignment` (`readAlignment`: reads land in -/// blocks), and the request is at most `max_read_request_size` and, with an -/// alignment, longer than two units, so a read widened by a unit at each -/// end fits. `zml.io.Loader.init` refuses options for which a block -/// calibration may select has none. -pub fn requestSize(read_chunk_size: usize, block_size: usize, alignment: usize) ?usize { - if (block_size == 0 or block_size % @max(alignment, 1) != 0) return null; - // Bounded before the rounding, which then cannot overflow. - const least = @max(read_chunk_size, block_size); - if (least > max_read_request_size) return null; - const request = std.mem.alignForwardAnyAlign(usize, least, block_size); - if (request > max_read_request_size) return null; - if (alignment != 0 and request <= 2 * alignment) return null; - return request; -} - -/// What the direct backend widens the reads of a file read directly to: -/// the VFS's alignment when `io` is a VFS and `direct_io` is not `off`, else -/// 0, every read exact. The VFS decides per local file whether it is read -/// directly (`VFS.useDirectIo`). -pub fn readAlignment(io: std.Io, direct_io: VFS.DirectIo) usize { - if (direct_io == .off or VFS.fromIo(io) == null) return 0; - return VFS.direct_io_alignment; -} - -/// `zml.io.Loader.Options` as `zml.io.Loader.init` checked them, the -/// checkpoint resolved to its VFS backend's read hints and the read width to -/// its default. See `zml.io.Loader.Options` for each field. -pub const Options = struct { - read_parallelism: usize, - read_hints: VFS.ReadHints = .local, - dma_block_size: ?usize = null, - direct_io: VFS.DirectIo = .auto, - /// The loader's checkpoint (loader.zig, `Loader.Options.checkpoint`) - /// when it is local and the VFS may read it direct (`readAlignment` is - /// not 0), else null; owned by the loader's set-up task. The direct - /// backend predicts from it how the checkpoint's files read, places its - /// arenas for that, and reads each of those files as predicted - /// (`direct_loader.Loader.create`); it keeps nothing of the path. - checkpoint: ?[]const u8 = null, - - /// The blocks calibration may select: `dma_block_size`, else - /// `dma_calibration.default_block_sizes`. - pub fn dmaBlockSizes(self: *const Options) []const usize { - return if (self.dma_block_size) |*block_size| block_size[0..1] else &dma_calibration.default_block_sizes; - } -}; - -/// One destination, its caller-owned output and the source ranges that fill -/// it, as the front end checked them (`checkParts`, `Backend.stridedRead`): -/// backends trust them. -/// Everything a spec names (its parts, their dimensions, the sources, the -/// output) stays alive until its await; backends borrow it. -pub const LoadSpec = struct { - /// The destination's global shape, which the output buffer keeps. Its - /// sharding places the bytes on the devices; a shape without a mesh is - /// replicated over every device (`Platform.placedShape`). - shape: Shape, - /// Where the bytes go, from the source's `TensorStore` binding. The - /// front end only submits kinds `supportsMemory` accepts, and a host - /// kind only with contiguous parts (`Lattice.dims` empty). - memory: Memory = .default, - /// Written whole once the destination's buffers exist, and not touched - /// before: a failure leaves it as it was or holding those buffers. - output: *Buffer, - /// What fills the destination's logical bytes, in ascending, disjoint - /// destination order; every byte no part covers is zero. - parts: []const LoadPart, -}; - -/// The bytes of `source` that `lattice` moves: its source offsets relative -/// to the tensor's data, its destination offsets into the destination's -/// logical bytes, the row-major bytes of its packed shape before the -/// sharding splits them. -pub const LoadPart = struct { - source: *const safetensors.Tensor, - lattice: Lattice, - - /// The whole of `source`, at the start of the destination. - pub fn whole(source: *const safetensors.Tensor) LoadPart { - return .{ .source = source, .lattice = .{ .width = source.byteSize() } }; - } -}; - -/// The most dimensions a part repeats over: `Source` gives each axis at -/// most two. -pub const max_part_dims = 2 * constants.MAX_RANK; - -/// A part's byte moves: the run of `width` bytes at `src_offset` and -/// `dst_offset`, and, for a strided part, one more for every index of -/// `dims` (outermost first), each offset moved by the index times its -/// strides. A zero source stride repeats the same bytes. Runs are numbered -/// row-major over `dims`. -pub const Lattice = struct { - src_offset: u64 = 0, - dst_offset: u64 = 0, - width: u64, - /// Empty for one contiguous run. - dims: []const Dim = &.{}, - - /// A step of the runs: `count` of them, each moving the source and - /// destination offsets by their strides. - pub const Dim = struct { - count: u64, - src_stride: u64, - dst_stride: u64, - }; - - pub const Run = struct { src: u64, dst: u64 }; - - pub fn runs(self: Lattice) u64 { - var n: u64 = 1; - for (self.dims) |d| n *= d.count; - return n; - } - - pub fn at(self: Lattice, index: u64) Run { - var rest = index; - var run: Run = .{ .src = self.src_offset, .dst = self.dst_offset }; - var d = self.dims.len; - while (d > 0) { - d -= 1; - const i = rest % self.dims[d].count; - rest /= self.dims[d].count; - run.src += i * self.dims[d].src_stride; - run.dst += i * self.dims[d].dst_stride; - } - return run; - } - - /// The source and destination bytes the runs span from their first - /// offsets, null when a count is zero or the bounds overflow. - pub fn spans(self: Lattice) ?struct { src: u64, dst: u64 } { - var src = self.width; - var dst = self.width; - for (self.dims) |d| { - if (d.count == 0) return null; - src = std.math.add(u64, std.math.mul(u64, d.count - 1, d.src_stride) catch return null, src) catch return null; - dst = std.math.add(u64, std.math.mul(u64, d.count - 1, d.dst_stride) catch return null, dst) catch return null; - } - return .{ .src = src, .dst = dst }; - } - - /// Whether each dimension steps past everything inside it on the - /// destination side: runs land in index order without overlapping. - /// Needs `spans` not null. - pub fn dstOrdered(self: Lattice) bool { - return ordered(self.dims, self.width, .dst, false); - } - - /// Whether the source runs come in index order without overlapping, - /// gaps allowed: the runs of an index range lie between the first's - /// start and the last's end. Needs `spans` not null. - pub fn srcOrdered(self: Lattice) bool { - return ordered(self.dims, self.width, .src, false); - } - - /// Whether the source runs are back to back in index order: an index - /// range reads one contiguous source range, each byte once. Needs - /// `spans` not null. - pub fn srcTiled(self: Lattice) bool { - return ordered(self.dims, self.width, .src, true); - } - - /// Whether each dimension steps past everything inside it on `side`, - /// exactly with `exact`. The extents it sums are bounded by `spans`. - fn ordered(dims: []const Dim, width: u64, comptime side: enum { src, dst }, exact: bool) bool { - var extent = width; - var d = dims.len; - while (d > 0) { - d -= 1; - if (dims[d].count < 2) continue; - const stride = if (side == .src) dims[d].src_stride else dims[d].dst_stride; - if (if (exact) stride != extent else stride < extent) return false; - extent += (dims[d].count - 1) * stride; - } - return true; - } - - /// The first run, in index order, that ends after destination offset - /// `offset`; `runs()` when none does. Needs `dstOrdered`. - pub fn firstEndingAfter(self: Lattice, offset: u64) u64 { - var low: u64 = 0; - var high = self.runs(); - while (low < high) { - const middle = low + (high - low) / 2; - if (self.at(middle).dst + self.width > offset) high = middle else low = middle + 1; - } - return low; - } - - /// The first run that starts at destination offset `offset` or later; - /// `runs()` when none does. Needs `dstOrdered`. - pub fn firstStartingFrom(self: Lattice, offset: u64) u64 { - var low: u64 = 0; - var high = self.runs(); - while (low < high) { - const middle = low + (high - low) / 2; - if (self.at(middle).dst >= offset) high = middle else low = middle + 1; - } - return low; - } - - /// The destination bytes of the runs inside `[start, end)`. - pub fn coveredIn(self: Lattice, start: u64, end: u64) u64 { - const first = self.firstEndingAfter(start); - const last = self.firstStartingFrom(end); - if (first >= last) return 0; - var covered = (last - first) * self.width; - const head = self.at(first).dst; - if (head < start) covered -= start - head; - const tail = self.at(last - 1).dst + self.width; - if (tail > end) covered -= tail - end; - return covered; - } - - /// Runs `first` to `end` (exclusive), in index order, without a - /// division per run. - pub const Cursor = struct { - lattice: Lattice, - index: u64, - end: u64, - counters: [max_part_dims]u64, - run: Run, - - pub fn init(lattice: Lattice, first: u64, end: u64) Cursor { - std.debug.assert(lattice.dims.len <= max_part_dims); - var cursor: Cursor = .{ .lattice = lattice, .index = first, .end = end, .counters = undefined, .run = lattice.at(first) }; - var rest = first; - var d = lattice.dims.len; - while (d > 0) { - d -= 1; - cursor.counters[d] = rest % lattice.dims[d].count; - rest /= lattice.dims[d].count; - } - return cursor; - } - - pub fn next(self: *Cursor) ?Run { - if (self.index >= self.end) return null; - const current = self.run; - self.index += 1; - if (self.index < self.end) { - const dims = self.lattice.dims; - var d = dims.len; - while (d > 0) { - d -= 1; - self.counters[d] += 1; - self.run.src +%= dims[d].src_stride; - self.run.dst +%= dims[d].dst_stride; - if (self.counters[d] < dims[d].count) break; - self.counters[d] = 0; - self.run.src -%= dims[d].count * dims[d].src_stride; - self.run.dst -%= dims[d].count * dims[d].dst_stride; - } - } - return current; - } - }; - - /// Copies runs `first` to `end` (exclusive): the `width` bytes at - /// `src + run.src` to `dst + run.dst`, `src` and `dst` being the - /// addresses the offsets count from (wrapping). Whole rows of the two - /// innermost dimensions go through nested loops, and a run of 1, 2, 4, - /// 8 or 16 bytes is one load and one store: a swizzle's 4-byte runs - /// cost a few cycles each instead of a cursor step and a `memcpy`. - pub fn copyRuns(self: Lattice, first: u64, end: u64, dst: usize, src: usize) void { - switch (self.width) { - inline 1, 2, 4, 8, 16 => |width| self.copyRunsOf(width, first, end, dst, src), - else => self.copyRunsOf(0, first, end, dst, src), - } - } - - /// `copyRuns` for runs of `width` bytes, or of `self.width` when zero. - fn copyRunsOf(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { - if (first >= end) return; - const inner = self.dims[self.dims.len - @min(self.dims.len, 2) ..]; - var row: u64 = 1; - for (inner) |d| row *= d.count; - const rows_start = std.math.divCeil(u64, first, row) catch unreachable; - const rows_end = end / row; - if (rows_start >= rows_end) return self.copyEach(width, first, end, dst, src); - self.copyEach(width, first, rows_start * row, dst, src); - const outer: Lattice = .{ .src_offset = self.src_offset, .dst_offset = self.dst_offset, .width = self.width, .dims = self.dims[0 .. self.dims.len - inner.len] }; - var rows: Cursor = .init(outer, rows_start, rows_end); - while (rows.next()) |start| { - const row_dst = dst +% start.dst; - const row_src = src +% start.src; - switch (inner.len) { - 0 => copyRun(width, self.width, row_dst, row_src), - 1 => copyLine(width, self.width, inner[0], row_dst, row_src), - else => { - var line_dst = row_dst; - var line_src = row_src; - for (0..inner[0].count) |_| { - copyLine(width, self.width, inner[1], line_dst, line_src); - line_dst +%= inner[0].dst_stride; - line_src +%= inner[0].src_stride; - } - }, - } - } - self.copyEach(width, rows_end * row, end, dst, src); - } - - fn copyEach(self: Lattice, comptime width: usize, first: u64, end: u64, dst: usize, src: usize) void { - if (first >= end) return; - var cursor: Cursor = .init(self, first, end); - while (cursor.next()) |run| copyRun(width, self.width, dst +% run.dst, src +% run.src); - } - - inline fn copyLine(comptime width: usize, len: u64, d: Dim, dst: usize, src: usize) void { - var to = dst; - var from = src; - for (0..d.count) |_| { - copyRun(width, len, to, from); - to +%= d.dst_stride; - from +%= d.src_stride; - } - } - - inline fn copyRun(comptime width: usize, len: u64, dst: usize, src: usize) void { - if (width == 0) { - const n: usize = @intCast(len); - @memcpy(@as([*]u8, @ptrFromInt(dst))[0..n], @as([*]const u8, @ptrFromInt(src))[0..n]); - } else { - const Word = @Int(.unsigned, 8 * width); - @as(*align(1) Word, @ptrFromInt(dst)).* = @as(*align(1) const Word, @ptrFromInt(src)).*; - } - } -}; - -/// Whether `parts` keep the `LoadSpec` contract for a destination of -/// `destination_bytes`: there is at least one, each is non-empty and inside -/// its source and the destination, and they come in ascending destination -/// order, each starting at or after the previous contiguous part's end or -/// strided part's start. Each part's bytes end in their file at least -/// `VFS.direct_io_alignment` below `maxInt(u64)`, so a read of them widened -/// to the alignment does not overflow. Strided parts must land in index -/// order (`Lattice.dstOrdered`); interleaving ones are only checked against -/// each other by the bytes they cover together, since only `Source` makes -/// them, disjoint by construction. -pub fn checkParts(parts: []const LoadPart, destination_bytes: u64) bool { - if (parts.len == 0) return false; - var floor: u64 = 0; - var covered: u64 = 0; - for (parts) |part| { - const lattice = part.lattice; - if (lattice.width == 0 or lattice.dims.len > max_part_dims) return false; - const spans = lattice.spans() orelse return false; - if (!lattice.dstOrdered()) return false; - const source_end = std.math.add(u64, lattice.src_offset, spans.src) catch return false; - const file_end = std.math.add(u64, part.source.offset, source_end) catch return false; - const end = std.math.add(u64, lattice.dst_offset, spans.dst) catch return false; - if (source_end > part.source.byteSize() or file_end > std.math.maxInt(u64) - VFS.direct_io_alignment) return false; - if (end > destination_bytes or lattice.dst_offset < floor) return false; - floor = if (lattice.dims.len == 0) end else lattice.dst_offset; - covered +|= lattice.runs() * lattice.width; - } - return covered <= destination_bytes; -} - -/// Whether `platform` loads through the direct backend. Transfer-manager -/// support and host pinning are independent. CPU implements byte-range -/// transfers from ordinary pages but not DmaMap, so it shares coalescing -/// and bounded blocks without DMA support. The whole-tensor buffered path -/// has neither: on one B70, interleaved HF Qwen3.5-4B loads took 10.9 to -/// 16.3 s direct against about 48 s buffered. Furiosa stages in its -/// plugin's `.host_pinned` buffers (SDK dma-buf memory, which the SDK -/// transfers without pinning pages per request). TPU, neuron and metal -/// stay buffered until their transfer path is checked on hardware. -pub fn loadsDirect(platform: *const Platform) bool { - return switch (platform.target) { - .cuda, .rocm, .oneapi, .cpu, .furiosa => true, - .tpu, .neuron, .metal => false, - }; -} - -/// Whether the loader can deliver into `kind` on `platform`. The host -/// kinds need the direct pipeline and a pinned host memory space on every -/// device. CPU loads through the direct backend, so it accepts them (its -/// plugin resolves every kind to its one memory space). -pub fn supportsMemory(platform: *const Platform, kind: Memory) bool { - if (kind == .default) return true; - if (!loadsDirect(platform)) return false; - for (platform.devices) |*device| { - if (device.memory(.host_pinned) == null) return false; - } - return true; -} - -/// Bytes per device, by `platform.devices` index: `Platform.init` refuses -/// more devices than it holds. -pub const DeviceBytes = [Platform.MAX_NUM_DEVICES]u64; - -/// The transfer implementation selected for the platform (`loadsDirect`). -pub const Backend = union(enum) { - direct: *direct_loader.Loader, - buffered: *buffered_loader.Loader, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Options, - ) !Backend { - return if (loadsDirect(platform)) - .{ .direct = try direct_loader.Loader.create(allocator, io, platform, opts) } - else - .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, opts) }; - } - - /// Publishes `specs`, and completes one item of `progress` per spec. - /// Fails with what the backend detected in the files, host memory and - /// PJRT, logged there (a plain allocation failure is `OutOfMemory`, - /// unlogged). - pub fn submit(self: Backend, specs: []const LoadSpec, progress: std.Progress.Node) !Submission { - return switch (self) { - inline else => |loader, tag| @unionInit(Submission, @tagName(tag), try loader.submit(specs, progress)), - }; - } - - /// `read`, the source bytes every job of `part` reads whole, widened to - /// its file's alignment (0 when its jobs read only their own range: - /// source runs back to back, a contiguous part included), and `limit`, - /// the most one read takes: the front end refuses a part whose `read` - /// passes `limit`. Needs `checkParts`. The direct backend's limit is its - /// smallest read, whichever block calibration picks: the checkpoint's - /// read chunk (8 MiB locally) rounded up to whole blocks of the smallest - /// it may pick, 2 MiB or a pinned `dma_block_size` - /// (`direct_loader.Loader.stridedRead`); the buffered backend reads any - /// part. - pub fn stridedRead(self: Backend, part: LoadPart) StridedRead { - return switch (self) { - .direct => |loader| loader.stridedRead(part), - .buffered => .{ .read = 0, .limit = std.math.maxInt(u64) }, - }; - } - - pub const StridedRead = struct { read: u64, limit: u64 }; - - /// Waits for and retires `submission`, which this backend published. - /// Its batch is dangling afterwards. - pub fn await(self: Backend, submission: Submission) !void { - return switch (self) { - inline else => |loader, tag| loader.awaitBatch(@field(submission, @tagName(tag))), - }; - } - - /// Fails the backend unless it already failed: queued work is dropped, - /// work in flight drains, and every later await returns the first error. - pub fn abort(self: Backend) void { - switch (self) { - inline else => |loader| loader.abort(), - } - } - - pub fn destroy(self: Backend) void { - switch (self) { - inline else => |loader| loader.destroy(), - } - } -}; - -/// A published batch of the backend that published it. -pub const Submission = union(enum) { - direct: *direct_loader.Batch, - buffered: *buffered_loader.Batch, -}; - -/// A random lattice whose runs land in order (`Lattice.dstOrdered`), its -/// source strides anything, zero (a broadcast) included. -fn randomLattice(random: std.Random, dims: []Lattice.Dim) Lattice { - const width = random.intRangeAtMost(u64, 1, 17); - var extent = width; - var d = dims.len; - while (d > 0) { - d -= 1; - dims[d] = .{ - .count = random.intRangeAtMost(u64, 1, 5), - .src_stride = random.intRangeAtMost(u64, 0, 40), - .dst_stride = extent + random.intRangeAtMost(u64, 0, 3) * random.intRangeAtMost(u64, 0, 4), - }; - extent = dims[d].dst_stride * (dims[d].count - 1) + extent; - } - return .{ .src_offset = random.intRangeAtMost(u64, 0, 9), .dst_offset = random.intRangeAtMost(u64, 0, 9), .width = width, .dims = dims }; -} - -test "the cursor walks a range's runs where at places them, and copyRuns copies them" { - var prng: std.Random.DefaultPrng = .init(0x5eed); - const random = prng.random(); - var source: [4096]u8 = undefined; - for (&source, 0..) |*byte, i| byte.* = @truncate(i * 7 + 1); - var expected: [4096]u8 = undefined; - var actual: [4096]u8 = undefined; - var dims_buffer: [5]Lattice.Dim = undefined; - for (0..2000) |_| { - var lattice = randomLattice(random, dims_buffer[0..random.intRangeAtMost(usize, 0, dims_buffer.len)]); - // The widths `copyRuns` moves as one word, and others; a wider run - // may overlap its neighbours, which no part does. - if (random.boolean()) lattice.width = ([_]u64{ 1, 2, 4, 8, 16 })[random.uintLessThan(usize, 5)]; - const spans = lattice.spans().?; - if (lattice.src_offset + spans.src > source.len or lattice.dst_offset + spans.dst > expected.len or !lattice.dstOrdered()) continue; - const runs = lattice.runs(); - const first = random.uintAtMost(u64, runs); - const end = random.intRangeAtMost(u64, first, runs); - @memset(&expected, 0); - @memset(&actual, 0); - var cursor: Lattice.Cursor = .init(lattice, first, end); - for (first..end) |index| { - const run = lattice.at(index); - try std.testing.expectEqual(run, cursor.next().?); - const width: usize = @intCast(lattice.width); - @memcpy(expected[@intCast(run.dst)..][0..width], source[@intCast(run.src)..][0..width]); - } - try std.testing.expectEqual(null, cursor.next()); - lattice.copyRuns(first, end, @intFromPtr(&actual), @intFromPtr(&source)); - try std.testing.expectEqualSlices(u8, &expected, &actual); - } -} - -test "checkParts refuses parts outside their source or destination, out of order or overlapping" { - // A source of 16 bytes into a destination of 32. - const source: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = 0 }; - // Sixteen bytes ending at the largest file offset a read widened to the - // alignment can reach, and one byte later. - const last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = std.math.maxInt(u64) - VFS.direct_io_alignment - 16 }; - const past_last: safetensors.Tensor = .{ .file_uri = "memory", .name = "s", .shape = .init(.{16}, .u8), .offset = last.offset + 1 }; - // Runs of 4 bytes, every other 4 bytes of the destination. - const every_other: []const Lattice.Dim = &.{.{ .count = 4, .src_stride = 4, .dst_stride = 8 }}; - const refused = [_][]const LoadPart{ - &.{}, - &.{.{ .source = &source, .lattice = .{ .width = 0 } }}, - // Past the source, past the destination, an overflow, and past the - // largest file offset. - &.{.{ .source = &source, .lattice = .{ .src_offset = 8, .width = 9 } }}, - &.{.{ .source = &source, .lattice = .{ .dst_offset = 24, .width = 9 } }}, - &.{.{ .source = &source, .lattice = .{ .src_offset = std.math.maxInt(u64), .width = 1 } }}, - &.{.whole(&past_last)}, - // Overlapping, and descending. - &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } } }, - &.{ .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4 } }, .{ .source = &source, .lattice = .{ .width = 4 } } }, - // Strided: runs that overlap on the destination, a dimension of no - // step, strides whose span overflows, past the source, past the - // destination, and descending. - &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 2 }} } }}, - &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 0, .src_stride = 4, .dst_stride = 8 }} } }}, - &.{.{ .source = &source, .lattice = .{ .width = 4, .dims = &.{.{ .count = 1 << 40, .src_stride = 0, .dst_stride = 1 << 40 }} } }}, - &.{.{ .source = &source, .lattice = .{ .src_offset = 4, .width = 4, .dims = every_other } }}, - &.{.{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = every_other } }}, - &.{ .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } } }, - // Interleaving strided parts that cover more than the destination, - // and contiguous parts overlapping beside a strided one. - &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, - &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, - }; - for (refused) |parts| try std.testing.expect(!checkParts(parts, 32)); - const accepted = [_][]const LoadPart{ - // Back to back. - &.{ .{ .source = &source, .lattice = .{ .width = 16 } }, .{ .source = &source, .lattice = .{ .dst_offset = 16, .width = 16 } } }, - // Two strided parts interleaving their runs. - &.{ .{ .source = &source, .lattice = .{ .width = 4, .dims = every_other } }, .{ .source = &source, .lattice = .{ .dst_offset = 4, .width = 4, .dims = every_other } } }, - // A contiguous part, then a strided one. - &.{ .{ .source = &source, .lattice = .{ .width = 8 } }, .{ .source = &source, .lattice = .{ .dst_offset = 8, .width = 4, .dims = &.{.{ .count = 2, .src_stride = 4, .dst_stride = 8 }} } } }, - // A broadcast: one source run repeated over the destination. - &.{.{ .source = &source, .lattice = .{ .width = 8, .dims = &.{.{ .count = 4, .src_stride = 0, .dst_stride = 8 }} } }}, - // Ending at the largest file offset. - &.{.whole(&last)}, - }; - for (accepted) |parts| try std.testing.expect(checkParts(parts, 32)); -} - -test "requestSize rounds the read chunk up to whole blocks of whole alignment units" { - const mib: usize = 1 << 20; - // One block at least, the chunk rounded up to whole blocks, and a - // rounded request past the largest refused. - try std.testing.expectEqual(16 * mib, requestSize(8 * mib, 16 * mib, 4096)); - try std.testing.expectEqual(9 * mib, requestSize(8 * mib, 3 * mib, 4096)); - try std.testing.expectEqual(null, requestSize(32 * mib, 24 * mib, 0)); - // Without an alignment any nonzero block; with one, only its multiples. - try std.testing.expectEqual(8 * mib + 2048, requestSize(8 * mib, 8 * mib + 2048, 0)); - try std.testing.expectEqual(null, requestSize(8 * mib, 8 * mib + 2048, 4096)); - try std.testing.expectEqual(null, requestSize(8 * mib, 0, 0)); - // A request of two alignment units cannot hold a read widened at both - // ends. - try std.testing.expectEqual(null, requestSize(4096, 8192, 4096)); - try std.testing.expectEqual(3 * 4096, requestSize(4096, 3 * 4096, 4096)); -} - -test "the options pin a DMA block or leave calibration its candidates, and only a VFS widens reads" { - const pinned: Options = .{ .read_parallelism = 1, .dma_block_size = 4 << 20 }; - try std.testing.expectEqualSlices(usize, &.{4 << 20}, pinned.dmaBlockSizes()); - const calibrated: Options = .{ .read_parallelism = 1 }; - try std.testing.expectEqualSlices(usize, &dma_calibration.default_block_sizes, calibrated.dmaBlockSizes()); - - var filesystem = try VFS.init(std.testing.allocator, std.testing.io); - defer filesystem.deinit(); - try std.testing.expectEqual(VFS.direct_io_alignment, readAlignment(filesystem.io(), .auto)); - try std.testing.expectEqual(0, readAlignment(filesystem.io(), .off)); - try std.testing.expectEqual(0, readAlignment(std.testing.io, .on)); -} diff --git a/zml/io/buffered_loader.zig b/zml/io/buffered_loader.zig deleted file mode 100644 index 96cfac3e..00000000 --- a/zml/io/buffered_loader.zig +++ /dev/null @@ -1,839 +0,0 @@ -//! Whole-tensor staging for platforms that use Buffer.from: TPU, neuron -//! and metal, until the direct path is checked on them. A composite -//! destination is assembled in the staging: its gaps zeroed, each -//! part read at its offset (a strided part reads its source range once and -//! copies each run where it lands), then the whole tensor is placed as any -//! other. Each batch runs up to the read width of workers over its specs; -//! read permits bound the source reads and one byte budget the host -//! staging, live and kept for reuse. There is no calibration and no pinned -//! memory. The shared front end owns source lookup, programs and admission. -const std = @import("std"); - -const stdx = @import("stdx"); -const VFS = @import("vfs"); - -const Buffer = @import("../buffer.zig").Buffer; -const Platform = @import("../platform.zig").Platform; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const backend = @import("backend.zig"); -const testing = @import("../testing.zig"); -const LoadPart = backend.LoadPart; -const LoadSpec = backend.LoadSpec; -const log = @import("log.zig").load; - -/// The host staging budget in largest requests: this many times the -/// largest staging request submitted (`stagingBytes`), for the buffers in -/// use and those kept for reuse alike. Twelve rather than the read width: -/// at a high-latency width of 32 the read width would nearly triple the -/// host staging for nothing this path can use (on such a source a tensor -/// splits its read across the free permits). -const staging_tensors: usize = 12; - -pub const Loader = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - /// Workers per batch, so the threads grow with the batches in flight: - /// the read permits bound their reads and the staging budget their - /// bytes, not their number. - width: usize, - /// The smallest claim of a tensor's read split across helpers - /// (`ChunkedRead.claim`). A tensor is one positional read unless spare - /// permits split it on a high-latency source. - read_chunk_size: usize, - /// The widest fan-out one tensor may take. One on a source whose reads - /// are bandwidth-bound rather than round-trip bound: splitting there only - /// spends tasks, and measurably so -- a replicated local load, whose long - /// transfers keep permits free, loses time to helpers it has no use for. - tensor_workers: usize, - permits: ReadPermits, - staging: Staging, - first_error: stdx.FirstError(anyerror) = .{}, - - pub fn create( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: backend.Options, - ) !*Loader { - const self = try allocator.create(Loader); - self.* = .{ - .allocator = allocator, - .io = io, - .platform = platform, - .width = opts.read_parallelism, - .staging = .{ .slots = @min(opts.read_parallelism, staging_tensors) }, - .read_chunk_size = opts.read_hints.read_chunk_size, - // A high-latency source may use spare read permits to split a - // tensor already staging: more concurrency without more host memory. - .tensor_workers = if (opts.read_hints.high_latency) opts.read_parallelism else 1, - .permits = .{ .free = opts.read_parallelism }, - }; - return self; - } - - /// Starts up to `width` workers over the specs, each assembling one - /// destination at a time from its parts. The batch borrows `specs` and - /// everything they name until it is awaited (`backend.LoadSpec`). On an - /// io that starts no worker, the batch loads inside this call. Nothing - /// runs when this fails. - pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { - try self.checkOpen(); - var largest: usize = 0; - for (specs) |spec| largest = @max(largest, stagingBytes(spec)); - const batch = try self.allocator.create(Batch); - batch.* = .{ .specs = specs, .progress = progress }; - self.staging.widen(self.io, largest); - // `concurrent`, never `async`: a busy Threaded io runs `async` - // inline, and an inline first worker would drain the whole cursor - // here. - var started: usize = 0; - for (0..@min(self.width, specs.len)) |_| { - batch.workers.concurrent(self.io, work, .{ self, batch }) catch break; - started += 1; - } - if (started == 0) self.work(batch); - return batch; - } - - /// Waits for the batch's workers, frees it and returns the sticky error. - /// Uncancelable: the workers borrow the batch and its specs until they - /// return. - pub fn awaitBatch(self: *Loader, batch: *Batch) !void { - const protection = self.io.swapCancelProtection(.blocked); - batch.workers.await(self.io) catch |err| switch (err) { - error.Canceled => unreachable, - }; - _ = self.io.swapCancelProtection(protection); - self.allocator.destroy(batch); - try self.checkOpen(); - } - - /// Fails the loader unless it already failed: workers start no other - /// tensor, and one waiting for its staging returns once it gets it. - pub fn abort(self: *Loader) void { - self.recordError(error.Canceled, "loader aborted", .{}); - } - - /// Every batch was awaited, so no worker runs. - pub fn destroy(self: *Loader) void { - self.staging.deinit(self.allocator); - self.allocator.destroy(self); - } - - fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { - if (self.first_error.set(err)) log.failure(err, fmt, args); - } - - fn checkOpen(self: *Loader) !void { - if (self.first_error.get()) |err| return err; - } - - /// One of the batch's workers: loads the next unclaimed spec until none - /// is left. - fn work(self: *Loader, batch: *Batch) void { - while (true) { - const index = batch.next.fetchAdd(1, .monotonic); - if (index >= batch.specs.len) return; - self.loadOne(batch.specs[index]); - // One item on the caller's node: a child node per tensor runs - // out of std.Progress's 127. - batch.progress.completeOne(); - } - } - - fn loadOne(self: *Loader, spec: LoadSpec) void { - if (self.first_error.get() != null) return; - std.debug.assert(spec.parts.len != 0); - const tensor_bytes = spec.shape.byteSize(); - const staging = self.staging.acquire(self.allocator, self.io, stagingBytes(spec)) catch |err| - return self.fail(err, "allocate staging for", spec, &spec.parts[0]); - defer self.staging.release(self.allocator, self.io, staging); - // The staging may have taken a while: a failure meanwhile leaves - // this tensor unstarted. - if (self.first_error.get() != null) return; - const tensor = staging[0..tensor_bytes]; - // Staging is reused, so a byte no part writes would publish another - // tensor's bytes rather than zero: the gaps are zeroed first. - var strided = false; - for (spec.parts) |part| strided = strided or part.lattice.dims.len != 0; - if (strided) { - @memset(tensor, 0); - } else { - // Only the gaps: the parts overwrite the rest. - var covered_end: usize = 0; - for (spec.parts) |part| { - const start: usize = @intCast(part.lattice.dst_offset); - @memset(tensor[covered_end..start], 0); - covered_end = start + @as(usize, @intCast(part.lattice.width)); - } - @memset(tensor[covered_end..], 0); - } - for (spec.parts) |*part| { - const file = std.Io.Dir.openFile(.cwd(), self.io, part.source.file_uri, .{ .mode = .read_only }) catch |err| - return self.fail(err, "open source for", spec, part); - defer file.close(self.io); - const lattice = part.lattice; - if (lattice.dims.len == 0) { - const start: usize = @intCast(lattice.dst_offset); - self.readInto(part.source, file, lattice.src_offset, tensor[start..][0..@intCast(lattice.width)]) catch |err| - return self.fail(err, "read", spec, part); - continue; - } - // A strided part: its source range once, into the staging past - // the tensor, then each run where it lands. - const source = staging[tensor_bytes..][0..@intCast(lattice.spans().?.src)]; - self.readInto(part.source, file, lattice.src_offset, source) catch |err| - return self.fail(err, "read", spec, part); - lattice.copyRuns(0, lattice.runs(), @intFromPtr(tensor.ptr), @intFromPtr(source.ptr) -% @as(usize, @intCast(lattice.src_offset))); - } - // The front end asks no host kind of this backend - // (`backend.supportsMemory`). - std.debug.assert(spec.memory == .default); - // The shape's sharding places the bytes; a shape without a mesh is - // replicated, as `Buffer.from` places it. - spec.output.* = Buffer.from(self.io, self.platform, spec.shape, tensor, .{ .wait = true }) catch |err| - return self.fail(err, "transfer", spec, &spec.parts[0]); - } - - /// Fails the loader with `err`, which `step` of `spec` met at `part`. - fn fail(self: *Loader, err: anyerror, step: []const u8, spec: LoadSpec, part: *const LoadPart) void { - self.recordError(err, "{s} tensor {s}: file={s}, offset={d}, source_bytes={d}, parts={d}, destination_bytes={d}", .{ step, part.source.name, part.source.file_uri, part.source.offset, part.source.byteSize(), spec.parts.len, spec.shape.byteSize() }); - } - - /// Fills `destination` from `source`'s bytes at `source_offset`, read - /// from `file`, splitting it across helper tasks when the source permits - /// are free. The permits are released before the caller's transfer, so a - /// tensor staging to the device lends its read budget to one still - /// reading. - fn readInto( - self: *Loader, - source: *const safetensors.Tensor, - file: std.Io.File, - source_offset: u64, - destination: []u8, - ) safetensors.Tensor.RangeError!void { - var chunked: ChunkedRead = .{ - .io = self.io, - .source = source, - .file = file, - .source_offset = source_offset, - .destination = destination, - .chunk_size = self.read_chunk_size, - }; - // Splitting below the read chunk buys nothing, so that is the - // most workers this tensor can use. - var useful_workers = @min( - self.tensor_workers, - destination.len / self.read_chunk_size + - @intFromBool(destination.len % self.read_chunk_size != 0), - ); - - self.permits.acquire(self.io); - var held: usize = 1; - var group: std.Io.Group = .init; - while (true) { - // Helpers only ever take permits that are already free, so a busy - // source sees one read per tensor. The check repeats between - // claims because the tensors that finish first hand their budget - // to the one still reading -- which is the long tensor that - // needed the help. - // Nothing unclaimed means a helper would only start and stop. - if (held < useful_workers and chunked.next.load(.monotonic) < destination.len) { - const extra = self.permits.tryAcquire(self.io, useful_workers - held); - if (extra != 0) { - // Claims size themselves by the width, so it is raised - // before a helper can claim. `concurrent`, since a busy - // io runs `async` inline, on this task. - chunked.workers.store(held + extra, .monotonic); - var started: usize = 0; - while (started < extra) : (started += 1) { - group.concurrent(self.io, ChunkedRead.run, .{&chunked}) catch break; - } - held += started; - if (started < extra) { - // The io starts no more: the rest of the permits go - // back, and this tensor stops asking. - self.permits.release(self.io, extra - started); - chunked.workers.store(held, .monotonic); - useful_workers = held; - } - } - } - if (!chunked.readOne()) break; - } - group.await(self.io) catch {}; - self.permits.release(self.io, held); - - if (chunked.failure.get()) |err| return err; - } -}; - -/// The staging one spec takes: its tensor, then room for its widest strided -/// part's source range, which is read once before its runs are copied out. -fn stagingBytes(spec: LoadSpec) usize { - var span: usize = 0; - for (spec.parts) |part| { - if (part.lattice.dims.len != 0) span = @max(span, @as(usize, @intCast(part.lattice.spans().?.src))); - } - return spec.shape.byteSize() + span; -} - -/// One buffered submission: its workers claim specs from `next`. -pub const Batch = struct { - specs: []const LoadSpec, - progress: std.Progress.Node, - next: std.atomic.Value(usize) = .init(0), - workers: std.Io.Group = .init, -}; - -/// Source reads in flight across every task of the buffered backend, at most -/// `read_parallelism`. On a high-latency source, a tensor wider than the -/// hints' `read_chunk_size` may split its read across the permits that are -/// free, for parallel streams. -const ReadPermits = struct { - mutex: std.Io.Mutex = .init, - available: std.Io.Condition = .init, - free: usize, - - fn acquire(self: *ReadPermits, io: std.Io) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (self.free == 0) self.available.waitUncancelable(io, &self.mutex); - self.free -= 1; - } - - /// Up to `count` permits, without waiting: how many were free is how much - /// concurrency this tensor gets, so a loaded source is left alone. - fn tryAcquire(self: *ReadPermits, io: std.Io, count: usize) usize { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - const taken = @min(count, self.free); - self.free -= taken; - return taken; - } - - fn release(self: *ReadPermits, io: std.Io, count: usize) void { - if (count == 0) return; - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - self.free += count; - for (0..count) |_| self.available.signal(io); - } -}; - -/// One tensor's read, claimed from a shared byte cursor by the owning task -/// and its helpers. Positional reads leave the file as it was, so they can -/// run concurrently on one file. -/// -/// A claim is the tensor split `workers` ways, and never smaller than the -/// hints' `read_chunk_size` -- that field is a minimum request size, and -/// on a high-latency source a request costs a round trip whether or not it -/// runs beside another. So one worker takes the whole tensor in a single -/// read, and only the concurrency the source can actually carry turns into -/// extra requests. -const ChunkedRead = struct { - io: std.Io, - source: *const safetensors.Tensor, - file: std.Io.File, - /// Where `destination` starts in the tensor's bytes. - source_offset: u64 = 0, - destination: []u8, - chunk_size: usize, - /// Reads this tensor may have in flight: the owner raises it before it - /// starts helpers on newly taken permits, and lowers it to the helpers - /// that started. - workers: std.atomic.Value(usize) = .init(1), - /// Bytes already claimed. - next: std.atomic.Value(usize) = .init(0), - failure: stdx.FirstError(safetensors.Tensor.RangeError) = .{}, - - fn run(self: *ChunkedRead) void { - while (self.readOne()) {} - } - - /// Reads the next unclaimed span. False once the tensor is fully claimed, - /// or once a read failed. - fn readOne(self: *ChunkedRead) bool { - if (self.failure.get() != null) return false; - const span = self.claim() orelse return false; - self.source.readRange(self.io, self.file, span.bytes, self.source_offset + span.offset) catch |err| { - _ = self.failure.set(err); - return false; - }; - return true; - } - - fn claim(self: *ChunkedRead) ?struct { offset: usize, bytes: []u8 } { - var start = self.next.load(.monotonic); - while (start < self.destination.len) { - const workers = self.workers.load(.monotonic); - const share = self.destination.len / workers + - @intFromBool(self.destination.len % workers != 0); - const len = @min(@max(self.chunk_size, share), self.destination.len - start); - if (self.next.cmpxchgWeak(start, start + len, .monotonic, .monotonic)) |actual| { - start = actual; - continue; - } - return .{ .offset = start, .bytes = self.destination[start..][0..len] }; - } - return null; - } -}; - -/// Host staging for the buffered backend: one budget for the buffers the -/// running tensors hold and those kept for reuse, which are handed back on -/// completion instead of freed. Mapping, faulting and then unmapping a whole -/// tensor's worth of pages for every tensor takes half of a serial buffered -/// load of a 14.96 GiB checkpoint, and a fifth of a concurrent one. -/// -/// The budget is `slots` times the largest request submitted, so no -/// request is wider than it. A request takes the smallest kept buffer that -/// fits, which counts in full while it is held: small tensors run far wider -/// than the slot count on small buffers, but once only large ones are kept -/// they take those, and run about `slots` wide. -const Staging = struct { - mutex: std.Io.Mutex = .init, - room: std.Io.Condition = .init, - /// The budget in largest requests (`staging_tensors`). - slots: usize, - /// Zero until the first submission reports its largest request. - budget: usize = 0, - /// What the running tensors hold. - live_bytes: usize = 0, - /// Buffers no tensor holds, and their total length. - free: std.ArrayList([]u8) = .empty, - retained_bytes: usize = 0, - - /// Raises the budget to `slots` of `largest`. Submissions only ever widen - /// it: a later submission of smaller tensors cannot shrink what is - /// already running. - fn widen(self: *Staging, io: std.Io, largest: usize) void { - const wanted = largest *| self.slots; - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - if (wanted <= self.budget) return; - self.budget = wanted; - self.room.broadcast(io); - } - - /// A whole allocation of at least `size` bytes, with undefined contents: - /// the allocation itself, not a slice of one. The caller slices it for - /// its own use and returns this exact slice to `release`, whatever - /// happens. Waits until a retained buffer fits, or `size` more bytes - /// fit the budget beside the live ones. `size` is at most the budget: - /// `widen` ran for the request's submission. - fn acquire(self: *Staging, allocator: std.mem.Allocator, io: std.Io, size: usize) ![]u8 { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - std.debug.assert(size <= self.budget); - while (true) { - if (self.take(size)) |buffer| return buffer; - if (self.live_bytes + size <= self.budget) break; - self.room.waitUncancelable(io, &self.mutex); - } - // No retained buffer fits, so each is smaller than this request: - // free them, smallest first, until it fits beside the rest. - while (self.live_bytes + self.retained_bytes + size > self.budget) { - const dropped = self.free.swapRemove(self.smallest()); - self.retained_bytes -= dropped.len; - allocator.free(dropped); - } - const fresh = try allocator.alloc(u8, size); - self.live_bytes += fresh.len; - std.debug.assert(self.live_bytes + self.retained_bytes <= self.budget); - return fresh; - } - - /// `buffer` must be what `acquire` returned, not a slice of it. It is - /// kept: the budget already counts it. - fn release(self: *Staging, allocator: std.mem.Allocator, io: std.Io, buffer: []u8) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - self.live_bytes -= buffer.len; - self.room.broadcast(io); - self.free.append(allocator, buffer) catch return allocator.free(buffer); - self.retained_bytes += buffer.len; - } - - /// Every tensor is done, so every buffer is back. - fn deinit(self: *Staging, allocator: std.mem.Allocator) void { - std.debug.assert(self.live_bytes == 0); - for (self.free.items) |buffer| allocator.free(buffer); - self.free.deinit(allocator); - } - - /// The smallest retained buffer that fits, now live, so the big ones - /// stay free for the big tensors. Called with `mutex` held. - fn take(self: *Staging, size: usize) ?[]u8 { - var best: ?usize = null; - for (self.free.items, 0..) |buffer, i| { - if (buffer.len < size) continue; - if (best == null or buffer.len < self.free.items[best.?].len) best = i; - } - const buffer = self.free.swapRemove(best orelse return null); - self.retained_bytes -= buffer.len; - self.live_bytes += buffer.len; - return buffer; - } - - /// Called with `mutex` held and a buffer retained. - fn smallest(self: *const Staging) usize { - var index: usize = 0; - for (self.free.items, 0..) |buffer, i| { - if (buffer.len < self.free.items[index].len) index = i; - } - return index; - } -}; - -test "chunked read splits a tensor only as wide as it has workers" { - var destination: [1000]u8 = undefined; - var work: ChunkedRead = .{ - .io = undefined, - .source = undefined, - .file = undefined, - .destination = &destination, - .chunk_size = 100, - }; - - // One worker reads the whole tensor, exactly as an unsplit read would. - const whole = work.claim().?; - try std.testing.expectEqual(@as(usize, 0), whole.offset); - try std.testing.expectEqual(@as(usize, 1000), whole.bytes.len); - try std.testing.expect(work.claim() == null); - - // Four workers take a quarter each. - work.next = .init(0); - work.workers = .init(4); - for (0..4) |i| { - const span = work.claim().?; - try std.testing.expectEqual(i * 250, span.offset); - try std.testing.expectEqual(@as(usize, 250), span.bytes.len); - } - try std.testing.expect(work.claim() == null); - - // Never below the read chunk, however many workers there are. - work.next = .init(0); - work.workers = .init(50); - const floored = work.claim().?; - try std.testing.expectEqual(@as(usize, 100), floored.bytes.len); -} - -test "staging holds live and retained bytes within one budget" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var staging: Staging = .{ .slots = 3 }; - defer staging.deinit(allocator); - - // Three slots of the largest request is the budget, and a later, - // smaller submission cannot shrink it. - staging.widen(io, 16); - staging.widen(io, 4); - try std.testing.expectEqual(@as(usize, 48), staging.budget); - - // With nothing kept, small requests run far wider than the slot count, - // and their buffers are kept. - var small: [12][]u8 = undefined; - for (&small) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); - try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); - for (small) |buffer| staging.release(allocator, io, buffer); - try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); - - // A request no retained buffer fits frees as many as it needs room - // from. - const large = try staging.acquire(allocator, io, 16); - try std.testing.expectEqual(@as(usize, 32), staging.retained_bytes); - staging.release(allocator, io, large); - // The smallest buffer that fits comes back to a smaller request, so the - // 16 stays for one the 4s cannot serve. The first take moves the 16 to - // the front of the list, ahead of the second's 4. - var reused: [2][]u8 = undefined; - for (&reused) |*buffer| { - buffer.* = try staging.acquire(allocator, io, 3); - try std.testing.expectEqual(@as(usize, 4), buffer.len); - } - try std.testing.expectEqual(@as(usize, 40), staging.retained_bytes); - for (reused) |buffer| staging.release(allocator, io, buffer); - try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); - - // A full budget holds the next request until a buffer comes back, then - // hands it that buffer. - var held: [3][]u8 = undefined; - for (&held) |*buffer| buffer.* = try staging.acquire(allocator, io, 16); - try std.testing.expectEqual(@as(usize, 48), staging.live_bytes + staging.retained_bytes); - var waiting = try io.concurrent(Staging.acquire, .{ &staging, allocator, io, 16 }); - try io.sleep(.fromMilliseconds(5), .awake); - try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); - staging.release(allocator, io, held[0]); - held[0] = try waiting.await(io); - try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); - for (held) |buffer| staging.release(allocator, io, buffer); - - // Once only large buffers are kept, small requests take them, each - // counted in full: three fill the budget. - var on_large: [3][]u8 = undefined; - for (&on_large) |*buffer| buffer.* = try staging.acquire(allocator, io, 4); - try std.testing.expectEqual(@as(usize, 48), staging.live_bytes); - for (on_large) |buffer| staging.release(allocator, io, buffer); - try std.testing.expectEqual(@as(usize, 48), staging.retained_bytes); -} - -test "buffered loader assembles a composite destination in reused staging" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 2); - defer devices.deinit(); - const platform = devices.platform; - const mesh = try devices.mesh("tp"); - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - var a: [48]u8 = undefined; - testing.pattern(&a, 1); - var b: [32]u8 = undefined; - testing.pattern(&b, 2); - var full: [128]u8 = undefined; - testing.pattern(&full, 3); - var big: [256]u8 = undefined; - testing.pattern(&big, 4); - try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ - .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &a }, - .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &b }, - .{ .name = "full", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &full }, - .{ .name = "big", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &big }, - }); - // Rows split over the two devices: b at row 1, a's last two rows (from - // byte 16 of a) at row 4, and a strided part, the left halves of full's - // rows 0 and 2, at rows 6 and 7, around a leading row, a gap and the - // right halves of zeros. - const shape = Shape.init(.{ .row = 8, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); - const columns = [_]backend.Lattice.Dim{.{ .count = 2, .src_stride = 32, .dst_stride = 16 }}; - const parts = [_]LoadPart{ - .{ .source = registry.tensors.getPtr("b").?, .lattice = .{ .dst_offset = 16, .width = 32 } }, - .{ .source = registry.tensors.getPtr("a").?, .lattice = .{ .src_offset = 16, .dst_offset = 64, .width = 32 } }, - .{ .source = registry.tensors.getPtr("full").?, .lattice = .{ .dst_offset = 96, .width = 8, .dims = &columns } }, - }; - try std.testing.expect(backend.checkParts(&parts, shape.byteSize())); - var expected: [128]u8 = @splat(0); - @memcpy(expected[16..48], &b); - @memcpy(expected[64..96], a[16..48]); - @memcpy(expected[96..104], full[0..8]); - @memcpy(expected[112..120], full[32..40]); - const whole = [_]LoadPart{.whole(registry.tensors.getPtr("full").?)}; - const big_shape = Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); - const big_whole = [_]LoadPart{.whole(registry.tensors.getPtr("big").?)}; - - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.destroy(); - // A bigger tensor first, so the composite, its strided source range - // included, reuses its staging buffer and only the zeroing keeps its - // bytes out of the gaps. - var first: Buffer = undefined; - try loader.awaitBatch(try loader.submit(&.{.{ .shape = big_shape, .output = &first, .parts = &big_whole }}, .none)); - defer first.deinit(); - try testing.expectShards(allocator, io, &first, &big, .default); - var output: Buffer = undefined; - try loader.awaitBatch(try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none)); - defer output.deinit(); - try testing.expectShards(allocator, io, &output, &expected, .default); - // One buffer was ever allocated, within the budget of two slots of the - // largest request. - try std.testing.expectEqual(@as(usize, 512), loader.staging.budget); - try std.testing.expectEqual(@as(usize, 256), loader.staging.retained_bytes); - - // A source whose file does not exist fails the worker, and the error - // is sticky from then on. - var missing = registry.tensors.get("full").?; - missing.file_uri = "missing.safetensors"; - const missing_parts = [_]LoadPart{.whole(&missing)}; - var never_written: Buffer = undefined; - const batch = try loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &missing_parts }}, .none); - try std.testing.expectError(error.FileNotFound, loader.awaitBatch(batch)); - // `expectError` would print the batch, and every type its specs name. - if (loader.submit(&.{.{ .shape = shape, .output = &never_written, .parts = &whole }}, .none)) |_| - return error.TestUnexpectedResult - else |err| - try std.testing.expectEqual(error.FileNotFound, err); -} - -test "a worker waiting for its staging loads nothing once the loader failed" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 1); - defer devices.deinit(); - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - var bytes: [64]u8 = undefined; - testing.pattern(&bytes, 1); - try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ - .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, - }); - const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); - const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; - - const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); - defer loader.destroy(); - // The test holds the whole budget, one slot of the tensor, so the - // worker waits for its staging until the loader failed. - loader.staging.widen(io, bytes.len); - const held = try loader.staging.acquire(allocator, io, bytes.len); - var output: Buffer = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; - defer output.deinit(); - const batch = try loader.submit(&.{.{ .shape = shape, .output = &output, .parts = &parts }}, .none); - try io.sleep(.fromMilliseconds(5), .awake); - loader.abort(); - loader.staging.release(allocator, io, held); - try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); - try std.testing.expectEqual(@as(usize, 0), output._shards.len); -} - -test "an aborted loader leaves each output as it was or whole, and stays failed" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 1); - defer devices.deinit(); - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - var bytes: [64]u8 = undefined; - testing.pattern(&bytes, 1); - try testing.writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ - .{ .name = "x", .dtype = "U8", .dims = &.{ 8, 8 }, .bytes = &bytes }, - }); - const shape = Shape.init(.{ .row = 8, .col = 8 }, .u8); - const parts = [_]LoadPart{.whole(registry.tensors.getPtr("x").?)}; - // One worker over many specs, so the abort lands among them. - var specs: [64]LoadSpec = undefined; - var outputs: [64]Buffer = undefined; - for (&specs, &outputs) |*spec, *output| { - output.* = .{ ._platform = devices.platform, ._shape = shape, ._shards = .empty }; - spec.* = .{ .shape = shape, .output = output, .parts = &parts }; - } - defer for (&outputs) |*output| output.deinit(); - - const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 1 }); - defer loader.destroy(); - const batch = try loader.submit(&specs, .none); - loader.abort(); - try std.testing.expectError(error.Canceled, loader.awaitBatch(batch)); - for (&outputs) |*output| { - if (output._shards.len != 0) try testing.expectShards(allocator, io, output, &bytes, .default); - } - if (loader.submit(&specs, .none)) |_| - return error.TestUnexpectedResult - else |err| - try std.testing.expectEqual(error.Canceled, err); -} - -test "buffered loader runs overlapping batches over their cursors, with or without concurrency" { - const allocator = std.testing.allocator; - var devices: testing.Devices = undefined; - try devices.init(allocator, std.testing.io, 2); - defer devices.deinit(); - const mesh = try devices.mesh("tp"); - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - // Tensors of 2 to 32 rows of 8 u16, so the staging budget holds a - // varying number of them. - const count = 24; - var bytes: [count][512]u8 = undefined; - var names: [count][8]u8 = undefined; - var dims: [count][2]i64 = undefined; - var entries: [count]testing.TensorBytes = undefined; - for (&bytes, &names, &dims, &entries, 0..) |*data, *name, *dim, *entry, i| { - const rows = 2 * (i % 16 + 1); - testing.pattern(data[0 .. rows * 16], @intCast(i + 1)); - dim.* = .{ @intCast(rows), 8 }; - entry.* = .{ .name = try std.fmt.bufPrint(name, "t{d}", .{i}), .dtype = "U16", .dims = dim, .bytes = data[0 .. rows * 16] }; - } - try testing.writeSafetensors(allocator, std.testing.io, tmp.dir, "weights.safetensors", ®istry, &entries); - - var parts: [count][1]LoadPart = undefined; - var specs: [count]LoadSpec = undefined; - var outputs: [count]Buffer = undefined; - for (&parts, &specs, &outputs, entries) |*part, *spec, *output, entry| { - part[0] = .whole(registry.tensors.getPtr(entry.name).?); - const shape = Shape.init(.{ .row = entry.dims[0], .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }); - spec.* = .{ .shape = shape, .output = output, .parts = part }; - } - - var no_concurrency: std.Io.Threaded = .init(allocator, .{ .async_limit = .nothing, .concurrent_limit = .nothing }); - defer no_concurrency.deinit(); - for ([_]std.Io{ std.testing.io, no_concurrency.io() }) |io| { - // A high-latency source with 32-byte chunks: helpers split each - // tensor whenever a permit is free. - const loader = try Loader.create(allocator, io, devices.platform, .{ - .read_parallelism = 4, - .read_hints = .{ .read_chunk_size = 32, .high_latency = true }, - }); - defer loader.destroy(); - // Both batches are in flight before either is awaited, each with - // its own workers: the permits bound their reads and the staging - // budget their bytes (`Staging.acquire` asserts it). - const first = try loader.submit(specs[0 .. count / 2], .none); - const second = try loader.submit(specs[count / 2 ..], .none); - try loader.awaitBatch(first); - try loader.awaitBatch(second); - defer for (&outputs) |*output| output.deinit(); - for (&outputs, entries) |*output, entry| try testing.expectShards(allocator, io, output, entry.bytes, .default); - try std.testing.expect(loader.staging.retained_bytes <= loader.staging.budget); - try std.testing.expectEqual(@as(usize, 0), loader.staging.live_bytes); - try std.testing.expectEqual(@as(usize, 4), loader.permits.free); - } -} - -test "a tensor from a high-latency HTTP source splits its read over the free permits" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 1); - defer devices.deinit(); - - var object: [4096]u8 = undefined; - testing.pattern(&object, 1); - var server = try VFS.MockServer.init(io, &object, .{}); - defer server.deinit(io); - try server.start(io); - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http }); - defer http.deinit(); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - try vfs.registerBackend("http", http.backend()); - - var url_buffer: [64]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); - const source: safetensors.Tensor = .{ .file_uri = url, .name = "x", .shape = .init(.{object.len}, .u8), .offset = 0 }; - const parts = [_]LoadPart{.whole(&source)}; - - // Four free permits and a quarter of the tensor as the read chunk: the - // worker and three helpers each read a quarter, one range GET each. - const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ - .read_parallelism = 4, - .read_hints = .{ .read_chunk_size = object.len / 4, .high_latency = true }, - }); - defer loader.destroy(); - var output: Buffer = undefined; - try loader.awaitBatch(try loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none)); - defer output.deinit(); - try server.stop(io); - try testing.expectShards(allocator, io, &output, &object, .default); - try std.testing.expectEqual(@as(usize, 4), server.get_requests.load(.acquire)); - try std.testing.expectEqual(@as(usize, 4), loader.permits.free); -} diff --git a/zml/io/direct_loader.zig b/zml/io/direct_loader.zig deleted file mode 100644 index a68e883b..00000000 --- a/zml/io/direct_loader.zig +++ /dev/null @@ -1,5361 +0,0 @@ -//! Direct loading follows a batch from per-file planning through FIFO claims, -//! source reads and DMA completion. Runtime ownership and synchronization -//! live here. CUDA, ROCm, oneAPI, Furiosa and CPU load this way -//! (`backend.loadsDirect`). -//! -//! Sizing, fixed by `Loader.create`. Calibration picks the DMA block -//! (dma_calibration.zig) and maps the ring it measures in; each device's -//! DMA budget is `dma_calibration.depth` blocks. A source request is the -//! checkpoint's minimum read (`VFS.readHints`) rounded up to whole blocks, -//! one block at least (`backend.requestSize`). The source width is -//! `read_parallelism`, clipped to what the mapped ceiling fits. The -//! workspace then maps, once and before the first read, what the ring -//! leaves of `width + 1` requests, the DMA reserve and the zero block, and -//! cuts them into blocks (`Workspace.carve`): nothing maps during a load. -//! `width + 1` workers share `width` read permits, so a full width of reads -//! is in flight while one worker queues what it read. The lifecycle credits -//! are the whole pinned set in requests, so transfers that still hold -//! blocks keep the DMA stage fed and never want a block the reads did not -//! leave free. -//! -//! Planning and claims. `submit` sorts a submission's parts by file and -//! offset, plans one file at a time and publishes each plan at once, so the -//! workers read the first file while the next is planned. A plan merges -//! touching or overlapping ranges into reads, never across a gap or a file, -//! and cuts them into jobs of at most a request; jobs keep that file order, -//! and the `Scheduler` hands them out in submission order. A worker takes a -//! credit, claims a job, reads it into its slot's pinned blocks and queues -//! its preplanned transfer pieces on per-device pumps. A block may feed -//! several outputs or devices; the request keeps its credit until every -//! piece it queued completed or was abandoned. The last-transfer flag goes -//! on the piece that completes a destination's bytes, so reads and DMA -//! complete in any order. A batch owns its plans and callback contexts -//! until every job completes; a failure retires the unclaimed jobs and -//! drains what is in flight before the batch goes. -//! -//! Direct I/O. A local file the VFS reads past the page cache -//! (`VFS.useDirectIo`, asked once per file: `Planner.Reads`) is read in -//! whole alignment units: each job's read widens to aligned bounds around -//! its range, into blocks that are whole units, and its transfers address -//! the widened read. A job's range is at most a request less two units, so -//! its widened read still fits its credit's slot. Every other file is read -//! exact. The arenas are placed for how the checkpoint reads, predicted -//! before calibration maps the first (`Planner.Reads.predict`): over every -//! allowed node when at least four fifths of its bytes will read direct -//! (host_memory.zig). The files predicted then read as predicted, and every -//! other file as its own decision says. -//! -//! Destinations. A part's source bytes map to its destination's logical -//! offsets, which the destination's `ShardGrid` places on the devices by -//! arithmetic. Pieces of one destination merge across parts when block, -//! writers and both offsets are contiguous, and its parts may come from -//! different files and plans. Bytes no part covers go out with the plan of -//! its first part, as a last fill job that reads nothing and transfers from -//! the zero block, which no credit owns. The job of a tensor sharded along -//! any axis but its first is staged by destination where that saves -//! submissions (`JobCut.placeDma`). Strided parts are gathered -//! (`JobCut.appendGatherJobs`), their runs copied out of the worker's bounce -//! buffer by `Lattice.copyRuns`. Block-scale swizzles gather this way, as -//! runs of four bytes. Runs under 4 KiB cost host copies, where a program -//! costs a compile and staging memory: on DeepSeek-V4.1 over two GB300, -//! laying the MXFP4 experts out here (gate and up rows interleaved, scales -//! swizzled) copied 77 to 189 GiB through bounce buffers, 17 to 36 s of -//! worker time; as programs it copied 1 to 3 GiB and loaded 0.1 to 1.1 s -//! faster, for 4 to 6 GiB more device memory at the peak. -//! -//! Host memory. A destination in a host memory space (`Memory.isHost`) -//! takes no DMA: the worker that reads its bytes writes them in place into -//! its PJRT host buffers, which an allocation task creates ahead of the -//! reads (`Loader.startAllocations`). A host job reads straight into its -//! destinations when its read is exact and addresses each byte once, in -//! order; a widened read, or parts that overlap, goes through its slot's -//! blocks and is copied from there. Bytes no part covers are zeroed in -//! place. The devices holding one box share one allocation, through views -//! of it (`SharedHostAllocation`), so a replicated destination is written -//! once. A `host_mapped` destination's shards are always such views, in -//! the devices' default memory space, so kernels read the pinned memory in -//! place. Host and DMA parts never share a job, and no strided part goes -//! to host memory. -//! -//! Locks. Where several pumps are held, they are taken in device order. A -//! pump's mutex may be held while a released request gives its credit back -//! (`Credits`), never the reverse, and no other lock is taken under the -//! scheduler's mutex. A DMA callback, on a plugin thread, holds one pump at -//! a time. - -const std = @import("std"); -const builtin = @import("builtin"); - -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); -const VFS = @import("vfs"); -const zio = @import("zio"); - -const Buffer = @import("../buffer.zig").Buffer; -const pjrtx = @import("../pjrtx.zig"); -const Platform = @import("../platform.zig").Platform; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const backend = @import("backend.zig"); -const BackendOptions = backend.Options; -const LoadPart = backend.LoadPart; -const LoadSpec = backend.LoadSpec; -const ShardGrid = @import("ShardGrid.zig"); -const dma_calibration = @import("dma_calibration.zig"); -const Memory = @import("TensorStore.zig").Memory; -const host_memory = @import("host_memory.zig"); -const load_log = @import("log.zig").load; -const testing = @import("../testing.zig"); - -/// Creating a tensor's device state: host memory or PJRT, since the front -/// end checked its placement and memory kind. `LazyOnce` replays it to every -/// later reader of the tensor. -const StateInitError = std.mem.Allocator.Error || pjrt.ApiError; - -/// Bounds per-device event overhead for tiny tensors. This is not a measured -/// optimum: 64 pieces smaller than block_size / 8 cannot fill an eight-block -/// byte budget. -const max_dma_pieces_per_device: usize = 64; - -/// The direct DMA backend. Submissions and awaits come from one task at a -/// time; the workers and the pumps run concurrently with them. -pub const Loader = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - /// The pinned arenas, grown once at creation and cut into `blocks`. - workspace: host_memory.Workspace, - /// The workspace's blocks (`carve`): the zero block first, then the - /// credits' slots (`Credits`). The zero block is the source of every DMA - /// fill transfer, in no credit's slot, never written again. - blocks: []const host_memory.Block, - scheduler: Scheduler, - metrics: Metrics = .{}, - /// One permit per source read in flight (`readSlot`), `read_width` of - /// them. The width never changes: a source that answers 429 (or 503 on - /// the object stores) is held one layer down, where the VFS holds every - /// request of that backend (`vfs/request.zig`), so the traffic falls - /// while the memory bounds stay. - read_permits: std.Io.Semaphore, - read_width: usize, - /// Lifecycle credits (`Credits`), beyond the workers' through the DMA - /// reserve. One credit beyond the width fed the DMA engines one request - /// at a time: on a GB300 loading DeepSeek-V4 (pieces of 4 MiB and 256 - /// KiB) that was 24.3 GiB/s at width 16 against 43.8 GiB/s with eight - /// requests queued, and the workers spent 9 ms per request waiting for a - /// credit against 3 ms reading. - credits: Credits, - /// The sticky first error (`recordError`). - first_error: stdx.FirstError(anyerror) = .{}, - /// One per device (`Dma.DevicePump`). Where several are locked at once - /// (`Dma.enqueue`, `Dma.retireBatch`) they are taken in device - /// order; a pump, a completion and `Dma.abortReady` hold one at a time. - pumps: []Dma.DevicePump, - /// Per-device in-flight budget: `dma_calibration.depth` blocks, in bytes. - dma_budget_bytes: usize, - worker_group: std.Io.Group = .init, - /// One per source path. Its file opens at first use (during planning - /// when a VFS decides direct IO for a local file, otherwise at the first - /// read) and stays open until `destroy`. Keyed by the slot's own copy of - /// the path: the registry that named it may go once the batch retired. - source_slots: std.StringHashMapUnmanaged(*SourceSlot) = .empty, - /// Device bytes of the outputs of the sealed batches, per - /// `platform.devices` index, cumulative, as `DestinationState.init` - /// allocates them; counted by the submitting task. - sealed_bytes: backend.DeviceBytes = @splat(0), - /// Device bytes allocated for outputs so far, per `platform.devices` - /// index, cumulative. Freeing an output does not decrease them. - allocated_bytes: [Platform.MAX_NUM_DEVICES]std.atomic.Value(u64) = @splat(.init(0)), - /// What the debug lines time from (`logBatch`, `logSummary`). - created_at: std.Io.Timestamp, - /// Submissions so far; the next batch's sequence number. - batch_count: usize = 0, - /// The block size, the request size and the gather limit, for the whole - /// load. - plan_config: Planner.Config, - /// How each file is read: exact, or aligned for direct I/O. - reads: Planner.Reads, - - pub fn create( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: BackendOptions, - ) !*Loader { - // A destination's shards and a transfer's writers (a u64 mask) are - // per device; Platform refuses devices past its maximum. - comptime std.debug.assert(Platform.MAX_NUM_DEVICES <= ShardGrid.max_boxes); - const self = try allocator.create(Loader); - errdefer allocator.destroy(self); - - var reads: Planner.Reads = .init(io, opts.direct_io); - errdefer reads.deinit(allocator); - // The arenas are placed for how the checkpoint reads before - // calibration maps the first: registered pages never move. - const probe_started: std.Io.Timestamp = .now(io, .awake); - const predicted = switch (platform.io_impl) { - .threaded => try reads.predict(allocator, opts.checkpoint), - // zio hands a file operation its event loop does not run (a - // real path; any, where it falls back to epoll) to a pool - // worker, and the compiles hold theirs for their whole run - // (`Compiler.compile`): the prediction takes a worker of its own - // rather than queue behind them, so the set-up overlaps them. - .zio => try zio.blockInPlaceReserved(Planner.Reads.predict, .{ &reads, allocator, opts.checkpoint }), - }; - if (opts.checkpoint != null) { - const probe_ns: u64 = @intCast(@max(probe_started.untilNow(io, .awake).nanoseconds, 0)); - load_log.debug("checkpoint reads predicted: files={d}, direct={Bi:.2} of {Bi:.2}, direct_reads={}, probe_ms={d:.3}", .{ - predicted.files, - predicted.direct_bytes, - predicted.bytes, - predicted.direct(), - @as(f64, @floatFromInt(probe_ns)) / std.time.ns_per_ms, - }); - } - var workspace: host_memory.Workspace = try .init(allocator, io, platform, predicted.direct()); - errdefer workspace.deinit(); - // At debug when canceled (`zml.io.Loader.deinit` cancels a set-up - // no `start` waited for) or out of memory (the workspace logs what - // it fails to map or pin). - const block_size = dma_calibration.calibrate(&workspace, platform, .{ .block_sizes = opts.dmaBlockSizes() }) catch |err| { - load_log.failure(err, "calibrate DMA: target={t}", .{platform.target}); - return err; - }; - const request_size = backend.requestSize(opts.read_hints.read_chunk_size, block_size, reads.alignment).?; - const blocks_per_request = request_size / block_size; - // The DMA stage of every device, kept mapped beside the source - // working set; at least one maximal source request, so a single - // request always fits beside it. - const dma_reserve = @max( - dma_calibration.depth * platform.devices.len, - Planner.blockCount(backend.max_read_request_size, block_size), - ); - // One more block for the zero fill, which no credit owns. - const zero_blocks = 1; - const available = workspace.capacity(block_size); - const width = @min( - opts.read_parallelism, - (available -| dma_reserve -| zero_blocks) / blocks_per_request -| 1, - ); - if (width == 0) { - load_log.err("host memory ceiling {Bi:.2} fits no source request beside the DMA reserve: available_blocks={d}, reserve_blocks={d}, blocks_per_request={d}: OutOfMemory", .{ host_memory.Workspace.mapped_bytes_ceiling, available, dma_reserve, blocks_per_request }); - return error.OutOfMemory; - } - if (width < opts.read_parallelism) { - load_log.debug("DMA source working set clipped by the mapped ceiling: width={d} of {d}", .{ - width, - opts.read_parallelism, - }); - } - // Two arenas, the reserve first: ROCm balances bytes across its host - // nodes per allocation, and that split is recorded. - const pregrowth_started: std.Io.Timestamp = .now(io, .awake); - const retained_before = workspace.mappedBytes(); - // The workspace logs what it fails to map. - const blocks = try workspace.carve(block_size, dma_reserve, (width + 1) * blocks_per_request + dma_reserve + zero_blocks); - const pregrowth_ns: u64 = @intCast(@max(pregrowth_started.untilNow(io, .awake).nanoseconds, 0)); - load_log.debug("host workspace pregrown: retained={Bi:.2}, pregrown={Bi:.2}, pregrowth_ms={d:.3}", .{ - workspace.mappedBytes(), - workspace.mappedBytes() - retained_before, - @as(f64, @floatFromInt(pregrowth_ns)) / std.time.ns_per_ms, - }); - std.debug.assert(blocks.len != 0); - @memset(blocks[0], 0); - var credits: Credits = try .init(allocator, blocks[1..], blocks_per_request); - errdefer credits.deinit(allocator); - const workers = width + 1; - std.debug.assert(credits.count() >= workers); - const pumps = try allocator.alloc(Dma.DevicePump, platform.devices.len); - errdefer allocator.free(pumps); - @memset(pumps, .{}); - - self.* = .{ - .allocator = allocator, - .io = io, - .platform = platform, - .workspace = workspace, - .blocks = blocks, - .scheduler = .init(allocator), - .read_permits = .{ .permits = width }, - .read_width = width, - .credits = credits, - .pumps = pumps, - .dma_budget_bytes = dma_calibration.depth * block_size, - .created_at = .now(io, .awake), - .plan_config = .{ - .block_size = block_size, - .request_size = request_size, - .gather_limit = backend.requestSize(opts.read_hints.read_chunk_size, std.mem.min(usize, opts.dmaBlockSizes()), reads.alignment).?, - }, - .reads = reads, - }; - std.debug.assert(self.dma_budget_bytes > 0); - errdefer self.scheduler.deinit(); - - errdefer self.stopWorkers(); - for (0..workers) |_| self.worker_group.concurrent(io, workerMain, .{self}) catch |err| { - load_log.err("start {d} source workers: {s}", .{ workers, @errorName(err) }); - return err; - }; - load_log.debug("live loader ready: target={s}, request_size={Bi:.2}, direct_io={t}, source_alignment={d}, dma_block_size={Bi:.2}, dma_budget_per_device={Bi:.2}, source_width={d}, lifecycle_credits={d}, workers={d}, retained={Bi:.2}", .{ - @tagName(platform.target), - request_size, - opts.direct_io, - reads.alignment, - block_size, - self.dma_budget_bytes, - width, - self.credits.count(), - workers, - self.workspace.mappedBytes(), - }); - return self; - } - - /// Plans `specs` one source file at a time, publishes each file's plan - /// as soon as it exists behind every earlier submission, then seals the - /// batch: work on the first file starts while the rest is planned. The - /// batch owns its destinations and parts until `awaitBatch` retires it. - /// The specs keep the `LoadSpec` contract and `stridedRead`'s limit, - /// which the front end checked. A planning failure fails the loader (a - /// partial submission can never complete): the batch is awaited here - /// and the caller sees only the first error. A failed loader refuses - /// every submit with its first error. The batch's host-resident - /// destinations are then created ahead of their reads - /// (`startAllocations`). - pub fn submit(self: *Loader, specs: []const LoadSpec, progress: std.Progress.Node) !*Batch { - try self.checkOpen(); - const batch = batch: { - const batch = try Batch.create(self.allocator, self.io, self.batch_count); - errdefer batch.destroy(); - try self.createDestinations(batch, specs, progress); - break :batch batch; - }; - Planner.publishFiles(&self.scheduler, self.io, batch, batch.parts, self.plan_config, self.reads) catch |err| { - // Fail the loader, seal and await what was published of this - // batch, and return the loader's first error. - self.recordError(err, "plan batch {d}, published_plans={d}", .{ batch.sequence, batch.plans.items.len }); - batch.sealed_at = .now(self.io, .awake); - self.batch_count += 1; - batch.releaseUnits(1); - try self.awaitBatch(batch); - unreachable; - }; - // Stamped before the sentinel goes, so before the batch can complete. - batch.sealed_at = .now(self.io, .awake); - self.batch_count += 1; - self.countSealed(batch); - self.startAllocations(batch); - // Every plan is visible: drop the publish sentinel. A batch without - // jobs completes right here. - batch.releaseUnits(1); - return batch; - } - - /// The source bytes every job of `part` reads whole, against one read - /// (`Planner.stridedRead`). - pub fn stridedRead(self: *const Loader, part: LoadPart) backend.Backend.StridedRead { - return Planner.stridedRead(self.plan_config, self.reads, part); - } - - /// Creates the batch's host-resident destinations ahead of their reads, - /// in claim order, on a task of its own. A first-time pinned allocation - /// runs at about 14-15 GiB/s on four B70 (oneAPI), serialized whatever - /// the number of callers (one to sixteen threads measured alike; the - /// time is the xe driver's VM_BIND), so on the workers' read path it - /// stalled them for 10-12 ms per tensor; ahead of them it overlaps the - /// reads. A worker that still finds its destination missing creates it - /// itself (`LazyOnce`). The task holds one completion unit, so the batch - /// outlives it; without a task the workers create everything. - fn startAllocations(self: *Loader, batch: *Batch) void { - var host_destinations: usize = 0; - for (batch.destinations) |*destination| host_destinations += @intFromBool(destination.memory.isHost()); - if (host_destinations == 0) return; - _ = batch.remaining.fetchAdd(1, .acq_rel); - self.worker_group.concurrent(self.io, allocateMain, .{ self, batch, host_destinations }) catch |err| { - load_log.debug("batch {d}: no allocation task, workers create its {d} host destinations: {s}", .{ batch.sequence, host_destinations, @errorName(err) }); - batch.releaseUnits(1); - }; - } - - /// Walks the host jobs in claim order (the plans in publish order, each - /// plan's jobs in order) and their transfers as `run` does, skipping a - /// destination already created. `ensured` also counts one a worker was - /// still creating, which the task waited for. - fn allocateMain(self: *Loader, batch: *Batch, host_destinations: usize) void { - var ensured: usize = 0; - var previous: ?*Destination = null; - walk: for (batch.plans.items) |plan| for (plan.jobs) |job| { - switch (job.kind) { - .host_direct, .host_bounce, .host_fill => {}, - .dma_read, .dma_land, .dma_fill => continue, - } - for (plan.transfersOf(job)) |transfer| { - const destination = transfer.destination; - if (destination == previous or destination.state.readyValue() != null) continue; - previous = destination; - if (self.first_error.get() != null) break :walk; - const started = awakeNs(self.io); - _ = destination.ensureState(self) catch |err| { - self.recordError(err, "initialize tensor {s} from {s} ahead of its reads, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); - break :walk; - }; - _ = self.metrics.ahead_init_ns.fetchAdd(awakeNs(self.io) -| started, .monotonic); - ensured += 1; - } - }; - load_log.debug("batch allocations: batch={d}, host_destinations={d}, ensured={d}, done=+{d:.3}s", .{ batch.sequence, host_destinations, ensured, secondsBetween(self.created_at, .now(self.io, .awake)) }); - batch.releaseUnits(1); - } - - /// Waits for the batch's last completion unit, retires it and returns - /// the loader's sticky error if the loader failed. Destroying the batch - /// destroys its transfer managers, and PJRT's manager destructor errors - /// every buffer whose last transfer never went out, so a buffer the - /// failure left open reports that error, never success. A transfer - /// that fails asynchronously errors its buffer's definition event too - /// (the done event carries no error). A program reading such a buffer - /// fails its execution; a bulk buffer shows the error at its first use. - /// Either way an output of a failed submission holds buffers whose - /// contents are unspecified, or was never written. A batch that - /// completed without an error submitted every byte of every target (the - /// pump flagged the submission that completed its bytes); one that did - /// not would leave a buffer PJRT then errors, a bookkeeping bug this - /// panics on. A host-resident destination has no target: its workers - /// write it in place, which only the debug check of its progress covers. - pub fn awaitBatch(self: *Loader, batch: *Batch) !void { - batch.done.waitUncancelable(self.io); - const done_at: std.Io.Timestamp = .now(self.io, .awake); - // Every request of this batch has completed: no worker or callback - // touches its managers or contexts any more. - const load_error = self.first_error.get(); - if (load_error == null) for (batch.destinations) |*destination| { - const submitted = submitted: { - const state = destination.state.readyValue() orelse break :submitted false; - for (state.targets) |target| { - if (!target.fullySubmitted()) break :submitted false; - } - break :submitted true; - }; - if (!submitted) std.debug.panic("loader: batch {d} completed without submitting every byte of tensor {s} {f}", .{ batch.sequence, destination.name(), destination.shape }); - }; - // Each destination completed its progress item. - if (load_error == null) for (batch.destinations) |*destination| std.debug.assert(destination.unread_bytes.load(.acquire) == 0); - Dma.retireBatch(self, batch); - self.logBatch(batch, done_at, load_error == null); - self.destroyBatch(batch); - if (load_error) |err| return err; - } - - /// Fails the loader unless it already failed (`recordError`): unclaimed - /// jobs retire, queued pieces drain, and requests in flight finish - /// their read. - pub fn abort(self: *Loader) void { - self.recordError(error.Canceled, "loader aborted", .{}); - } - - /// The caller awaited every batch before this; nothing is queued or in - /// flight, so stopping the workers is a plain shutdown. - pub fn destroy(self: *Loader) void { - self.stopWorkers(); - self.logSummary(); - var slots = self.source_slots.valueIterator(); - while (slots.next()) |slot| { - slot.*.deinit(self.io); - self.allocator.free(slot.*.uri); - self.allocator.destroy(slot.*); - } - self.source_slots.deinit(self.allocator); - self.reads.deinit(self.allocator); - std.debug.assert(self.credits.inUse(self.io) == 0); - for (self.pumps) |*device_pump| { - std.debug.assert(device_pump.active_pieces == 0); - std.debug.assert(device_pump.queue.len == 0); - std.debug.assert(device_pump.retired == null); - device_pump.queue.deinit(self.allocator); - } - self.allocator.free(self.pumps); - self.scheduler.deinit(); - self.credits.deinit(self.allocator); - self.workspace.deinit(); - self.allocator.destroy(self); - } - - /// Output bytes of the sealed batches not allocated yet, per - /// `platform.devices` index. Workers allocate as they read, possibly - /// before the batch is sealed, so the difference saturates. - pub fn unallocatedBytes(self: *const Loader) backend.DeviceBytes { - var unallocated = self.sealed_bytes; - for (&unallocated, &self.allocated_bytes) |*bytes, *allocated| bytes.* -|= allocated.load(.acquire); - return unallocated; - } - - /// Counts the device bytes of the sealed `batch`: a box of each - /// destination outside host memory on each of its devices, as - /// `DestinationState.init` allocates them. - fn countSealed(self: *Loader, batch: *const Batch) void { - for (batch.destinations) |*destination| { - if (destination.memory.isHost()) continue; - for (destination.grid.devices) |device| self.sealed_bytes[device.id] += destination.grid.box_bytes; - } - } - - fn checkOpen(self: *Loader) !void { - if (self.first_error.get()) |err| return err; - } - - /// Creates the batch's destinations, one per spec, and their parts in - /// its arena, their sources and dimensions staying the specs' until the - /// batch is awaited (`backend.LoadSpec`). A destination's device state - /// is created lazily by the first worker that reads or fills for it, - /// or by the allocation task. - fn createDestinations( - self: *Loader, - batch: *Batch, - specs: []const LoadSpec, - progress: std.Progress.Node, - ) !void { - const arena = batch.arena.allocator(); - var part_count: usize = 0; - for (specs) |spec| part_count += spec.parts.len; - const destinations = try arena.alloc(Destination, specs.len); - const parts = try arena.alloc(Part, part_count); - var next_part: usize = 0; - for (specs, destinations) |spec, *destination| { - const grid: ShardGrid = .init(self.platform, spec.shape); - const destination_parts = parts[next_part..][0..spec.parts.len]; - next_part += spec.parts.len; - var covered_bytes: u64 = 0; - var gathered = false; - for (spec.parts, destination_parts) |part, *slot| { - slot.* = .{ - .source = part.source, - .source_slot = try self.sourceSlot(part.source.file_uri), - .destination = destination, - .lattice = part.lattice, - }; - covered_bytes += part.lattice.width; - gathered = gathered or part.lattice.dims.len != 0; - const strided = self.stridedRead(part); - std.debug.assert(strided.read <= strided.limit); // backend.Backend.stridedRead - } - std.debug.assert(!gathered or !spec.memory.isHost()); // backend.LoadSpec.memory - destination.* = .{ - .shape = spec.shape, - .grid = grid, - .memory = spec.memory, - .output = spec.output, - .parts = destination_parts, - .gathered = gathered, - // A gathered destination's jobs land every byte, zeros too. - .unread_bytes = .init(if (gathered) grid.total else covered_bytes), - .progress = progress, - }; - } - batch.destinations = destinations; - batch.parts = parts; - } - - /// Releases the destinations' device state, then the batch's arena. - fn destroyBatch(self: *Loader, batch: *Batch) void { - for (batch.destinations) |*destination| destination.deinit(self.allocator, self.platform.pjrt_api); - batch.destroy(); - } - - fn sourceSlot(self: *Loader, uri: []const u8) !*SourceSlot { - if (self.source_slots.get(uri)) |slot| return slot; - const owned = try self.allocator.dupe(u8, uri); - errdefer self.allocator.free(owned); - const slot = try self.allocator.create(SourceSlot); - errdefer self.allocator.destroy(slot); - slot.* = .{ .uri = owned }; - try self.source_slots.putNoClobber(self.allocator, owned, slot); - return slot; - } - - fn workerMain(self: *Loader) void { - var scratch = ReadRequest.Scratch.init(self.allocator, self.credits.blocks_per_slot, self.plan_config.block_size) catch |err| { - self.recordError(err, "allocate source worker scratch: blocks={d}", .{self.credits.blocks_per_slot}); - return; - }; - defer scratch.deinit(); - while (self.scheduler.waitForWork(self.io)) { - const request = self.claimRequest() orelse continue; - // The scheduling sentinel keeps the batch, and with it the - // request's plan, alive through `run` and error reporting. - defer request.release(); - request.run(&scratch) catch |err| { - self.recordError(err, "source request: file={s}, job={t}", .{ request.plan.source_slot.uri, request.plan.jobs[request.index].kind }); - }; - } - } - - /// Takes a credit, then the next job (`Credits`): a claimed job never - /// waits for its slot. Null, with the credit given back, when nothing - /// was left to claim. - fn claimRequest(self: *Loader) ?*ReadRequest { - const credit_wait_started = awakeNs(self.io); - const slot = self.credits.acquire(self.io); - const credit_wait_ns = awakeNs(self.io) -| credit_wait_started; - const claim = self.scheduler.claim(self.io) orelse { - self.credits.release(self.io, slot); - return null; - }; - _ = self.metrics.credit_wait_ns.fetchAdd(credit_wait_ns, .monotonic); - return ReadRequest.init(self, claim, slot); - } - - /// Stops the workers: every batch was awaited, so nothing is queued and - /// no request holds a credit or permit, and `Scheduler.fail` only wakes - /// the workers waiting for work. A worker may still hold the credit it - /// took for a claim that finds nothing (`claimRequest`); it gives it - /// back before it sees `stopping`, so every credit is free once this - /// returns. Awaiting a group that never spawned returns at once, so - /// this also cleans up a failed `create`. - fn stopWorkers(self: *Loader) void { - std.debug.assert(self.scheduler.queue.len == 0); - self.scheduler.fail(self.io); - self.worker_group.await(self.io) catch {}; - } - - /// Keeps `err` as the sticky error unless one is kept already and logs - /// it, then fails the load without waiting: unclaimed jobs retire - /// (`Scheduler.fail`) and queued pieces are released - /// (`Dma.abortReady`). Any thread may call it, a DMA callback included; - /// requests in flight stop at their next read (`readSlot`) or enqueue - /// and release their units. - /// Failing other batches than the caller's is safe from any thread: - /// `Scheduler.fail` and `Dma.abortReady` reach only live batches (their - /// docs), and `destroy` waits for the worker group before it frees what - /// the workers use. - fn recordError(self: *Loader, err: anyerror, comptime fmt: []const u8, args: anytype) void { - if (self.first_error.set(err)) { - load_log.failure(err, fmt, args); - self.scheduler.fail(self.io); - Dma.abortReady(self); - } - } - - /// Counts the plans' jobs, transfers and events only when the lines - /// would be logged. - fn logBatch(self: *Loader, batch: *const Batch, done_at: std.Io.Timestamp, successful: bool) void { - if (!std.log.logEnabled(.debug, .@"zml/io/load")) return; - const sealed_at = batch.sealed_at orelse self.created_at; - const published_at = batch.published_at orelse sealed_at; - var logical_bytes: usize = 0; - for (batch.destinations) |*destination| logical_bytes += destination.shape.byteSize(); - load_log.debug("batch completed: batch={d}, successful={}, logical_bytes={Bi:.2}, published=+{d:.3}s, sealed=+{d:.3}s, done=+{d:.3}s, elapsed={d:.3}s, source_width={d}, request_size={Bi:.2}", .{ - batch.sequence, - successful, - logical_bytes, - secondsBetween(self.created_at, published_at), - secondsBetween(self.created_at, sealed_at), - secondsBetween(self.created_at, done_at), - secondsBetween(published_at, done_at), - self.read_width, - self.plan_config.request_size, - }); - - var planning_ns: u64 = 0; - var source_bytes: u64 = 0; - var jobs: usize = 0; - var transfers: usize = 0; - var dma_submissions: usize = 0; - var kinds: std.EnumArray(std.meta.Tag(Plan.Job.Kind), usize) = .initFill(0); - var gather_jobs: usize = 0; - for (batch.plans.items) |plan| { - planning_ns += plan.planning_ns; - source_bytes += plan.source_bytes; - jobs += plan.jobs.len; - transfers += plan.transfers.len; - dma_submissions += plan.events.len; - for (plan.jobs) |job| { - kinds.getPtr(job.kind).* += 1; - // A gathered destination's landing jobs land it alone. - if (job.kind == .dma_land and plan.transfersOf(job)[0].destination.gathered) gather_jobs += 1; - } - } - const fill_jobs = kinds.get(.dma_fill) + kinds.get(.host_fill); - const host_jobs = kinds.get(.host_direct) + kinds.get(.host_bounce) + kinds.get(.host_fill); - const source_jobs = jobs - fill_jobs; - const average_read_size = if (source_jobs == 0) 0 else source_bytes / source_jobs; - const coalescing_ratio = if (source_jobs == 0) - 0 - else - @as(f64, @floatFromInt(batch.parts.len)) / - @as(f64, @floatFromInt(source_jobs)); - load_log.debug("batch planning: batch={d}, plans={d}, planning_elapsed={d:.3}s, planned_source_bytes={Bi:.2}, planned_source_jobs={d}, fill_jobs={d}, host_jobs={d}, source_items={d}, planned_transfers={d}, planned_dma_submissions={d}, staged_jobs={d}, gather_jobs={d}, coalescing_ratio={d:.2}, average_read_size={Bi:.2}", .{ - batch.sequence, - batch.plans.items.len, - @as(f64, @floatFromInt(planning_ns)) / std.time.ns_per_s, - source_bytes, - source_jobs, - fill_jobs, - host_jobs, - batch.parts.len, - transfers, - dma_submissions, - kinds.get(.dma_land) - gather_jobs, - gather_jobs, - coalescing_ratio, - average_read_size, - }); - } - - fn logSummary(self: *Loader) void { - const reads = self.metrics.read_operations.load(.acquire); - load_log.debug("loader summary: batches={d}, successful={}, read_bytes={Bi:.2}, elapsed={d:.3}s, reads={d}, source_width={d}, request_size={Bi:.2}, credits_high_water={d} of {d}, pinned_mapped={Bi:.2}", .{ - self.batch_count, - self.first_error.get() == null, - self.metrics.read_bytes.load(.acquire), - secondsBetween(self.created_at, .now(self.io, .awake)), - reads, - self.read_width, - self.plan_config.request_size, - self.credits.high_water, - self.credits.count(), - self.workspace.mappedBytes(), - }); - load_log.debug("loader waits: credit_wait_ms_per_read={d:.3}, read_ms_per_read={d:.3}, dma_stage_ms_per_read={d:.3}, tensor_init_ms_per_read={d:.3}, ahead_init_ms={d:.3}", .{ - millisecondsPer(self.metrics.credit_wait_ns.load(.acquire), reads), - millisecondsPer(self.metrics.read_ns.load(.acquire), reads), - millisecondsPer(self.metrics.dma_stage_ns.load(.acquire), reads), - millisecondsPer(self.metrics.tensor_init_ns.load(.acquire), reads), - millisecondsPer(self.metrics.ahead_init_ns.load(.acquire), 1), - }); - const submissions = self.metrics.dma_submissions.load(.acquire); - load_log.debug("loader DMA: dma_submissions={d}, dma_submit_us_per_piece={d:.2}, dma_piece_latency_ms={d:.3}, pump_stops_empty={d}, pump_stops_full={d}", .{ - submissions, - millisecondsPer(self.metrics.dma_submit_ns.load(.acquire), submissions) * 1000, - millisecondsPer(self.metrics.dma_piece_ns.load(.acquire), submissions), - self.metrics.pump_stops_empty.load(.acquire), - self.metrics.pump_stops_full.load(.acquire), - }); - } -}; - -/// One submission: its plans (one per source file, published as each file -/// is planned), the destinations it writes and their parts, and every -/// request and event context created while loading it, all in the batch's -/// arena until it retires. `remaining` counts completion units: one per -/// published job plus a publish sentinel held until the submission is -/// sealed, and one for the allocation task while it runs -/// (`Loader.startAllocations`), taken before the sentinel is dropped. A -/// job's unit is released exactly once, by whichever of these happens: its -/// request's last reference drops (final DMA callback, a host job's end, or -/// abandonment), or `Scheduler.fail` retires it unclaimed. The batch is -/// done when `remaining` reaches zero; the awaiting task then retires it, -/// so releasing a unit is the last permitted access to the batch. -pub const Batch = struct { - /// Holds the plans with their contexts, the destinations and the parts - /// until `destroy`; only the submitting task allocates in it. - arena: std.heap.ArenaAllocator, - io: std.Io, - /// Published plans in file order: appended by the submitting task under - /// the scheduler mutex. - plans: std.ArrayList(*Plan) = .empty, - /// Their device state is the loader's, released before `destroy`. - destinations: []Destination = &.{}, - /// The destinations' parts, each destination's consecutive and in its - /// destination order: what the planner sorts by file offset. - parts: []Part = &.{}, - remaining: std.atomic.Value(usize), - done: std.Io.Event = .unset, - /// Submission number within the loader, for log correlation. - sequence: usize, - /// The first publish; null for a submission without plans. - published_at: ?std.Io.Timestamp = null, - /// Once every plan is published, before the publish sentinel goes. - sealed_at: ?std.Io.Timestamp = null, - - /// An open batch holding its publish sentinel and nothing else. The - /// sentinel keeps the batch from completing before its last plan is - /// visible. - fn create(allocator: std.mem.Allocator, io: std.Io, sequence: usize) !*Batch { - const self = try allocator.create(Batch); - self.* = .{ - .arena = .init(allocator), - .io = io, - .remaining = .init(1), - .sequence = sequence, - }; - return self; - } - - /// Releases `count` completion units. MEMORY-ORDER RULE: this must be the - /// caller's final access to the batch and to anything it owns (requests, - /// events, destinations, parts, plans; a request's slot of blocks is the - /// loader's and went back with its credit just before). The last unit - /// sets `done`, and the awaiting task frees all of it as soon as it - /// observes the event. - fn releaseUnits(self: *Batch, count: usize) void { - if (count == 0) return; - const previous = self.remaining.fetchSub(count, .acq_rel); - std.debug.assert(previous >= count); - if (previous == count) self.done.set(self.io); - } - - /// Frees the arena: the plans with their contexts, which - /// `Dma.retireBatch` must already have retired, the destinations and - /// the parts. - fn destroy(self: *Batch) void { - const allocator = self.arena.child_allocator; - self.arena.deinit(); - allocator.destroy(self); - } -}; - -/// One output buffer and what places it. Owned by the batch. Its parts may -/// come from several files, so several plans can write it: their pieces, -/// and the zero fill, all count toward the same targets (`Target.total` is -/// the placement's bytes, whichever plan submits them), and its device -/// state is created once, by the first worker that reads or fills for it -/// or, for a host-resident one, normally by the allocation task ahead of -/// them. -const Destination = struct { - /// The caller's shape, which the output buffer keeps. - shape: Shape, - /// Where `shape`, packed and placed (`Platform.placedShape`), lands on - /// its devices: what PJRT allocates and the planner cuts. - grid: ShardGrid, - /// Where its bytes go (`TensorStore.Memory`). - memory: Memory = .default, - output: *Buffer, - /// What fills it, in ascending, disjoint destination order - /// (`backend.LoadSpec`): a slice of the batch's parts. Bytes no part - /// covers are written from the loader's zero block, by the fill job of - /// the plan holding `parts[0]`. - parts: []Part, - /// Fed by strided parts: planned whole, by logical ranges, with the - /// plan holding `parts[0]` (`Planner.JobCut.appendGatherJobs`), whose - /// jobs also write its zero fill. - gathered: bool = false, - /// Bytes not read yet: the parts' widths, or for a gathered destination - /// its whole placed size (its gather jobs land the zeros too). The read - /// that brings it to zero completes the destination's item on - /// `progress`. - unread_bytes: std.atomic.Value(u64), - progress: std.Progress.Node = .none, - state: LazyOnce(DestinationState) = .{}, - - /// How its parts are planned, a class at a time in this order - /// (`Planner.sortedPartOrder`): DMA jobs, host jobs, or gathered whole. - const Delivery = enum { dma, host, gathered }; - - fn delivery(self: *const Destination) Delivery { - if (self.gathered) return .gathered; - return if (self.memory.isHost()) .host else .dma; - } - - fn ensureState(self: *Destination, direct: *Loader) StateInitError!*DestinationState { - return self.state.ensure(direct.io, DestinationState.init, .{ direct, self }); - } - - fn deinit(self: *Destination, allocator: std.mem.Allocator, api: *const pjrt.Api) void { - if (self.state.readyValue()) |state| state.deinit(allocator, api); - } - - /// The first source's name, for logs. - fn name(self: *const Destination) []const u8 { - return self.parts[0].source.name; - } - - /// Counts `bytes` as read (`unread_bytes`); the last read completes the - /// destination's one progress item. - fn recordRead(self: *Destination, bytes: u64) void { - if (self.unread_bytes.fetchSub(bytes, .acq_rel) == bytes) self.progress.completeOne(); - } -}; - -/// One source range of a destination: what the planner sorts by file offset -/// and cuts into jobs. A plain tensor is one part covering its whole -/// destination. -const Part = struct { - source: *const safetensors.Tensor, - source_slot: *SourceSlot, - destination: *Destination, - /// The spec part's; its dimensions are borrowed until the await - /// (`backend.LoadSpec`). - lattice: backend.Lattice, - - /// The part's bytes in its file, which `backend.checkParts` bounds. - fn fileRange(self: *const Part) struct { start: u64, end: u64 } { - const start = self.source.offset + self.lattice.src_offset; - return .{ .start = start, .end = start + self.lattice.width }; - } -}; - -/// One source file: every job of a plan reads the file of its slot, which -/// the first read opens, or the planner when it asks the VFS whether the -/// file is read directly (`Planner.Reads`). -const SourceSlot = struct { - uri: []const u8, - file: LazyOnce(std.Io.File) = .{}, - /// The file's read alignment once the planner decided it - /// (`Planner.Reads`): 0 when it is read exactly. - alignment: ?usize = null, - - fn ensure(self: *SourceSlot, io: std.Io) std.Io.File.OpenError!std.Io.File { - const file = try self.file.ensure(io, std.Io.Dir.openFile, .{ .cwd(), io, self.uri, .{ .mode = .read_only } }); - return file.*; - } - - fn deinit(self: *SourceSlot, io: std.Io) void { - if (self.file.readyValue()) |file| file.close(io); - } -}; - -/// One file's jobs in claim order with their transfer records and every -/// context the jobs can need, allocated by the planner: one request per -/// job and one event per planned DMA submission, handed out in -/// submission order; a host job needs only its request. Contexts hold the -/// plan's address, which the batch's arena keeps until it retires. -const Plan = struct { - /// A claimable job: its slice of the plan's transfers and what its - /// worker does for them. Its request slot is its index. - const Job = struct { - /// Where the job reads first: its read's start, a landing job's - /// first extent's; zero for a job that reads nothing. - /// `Planner.preparePlan` sorts the other jobs by it and appends - /// the fill jobs after them. - file_offset: u64, - transfer_start: usize, - transfer_len: usize, - kind: Kind, - - /// How the job's bytes reach their destinations. A destination - /// in a host memory space (`Memory.isHost`) is written - /// by the worker that reads it: going through pinned blocks and - /// a host-to-host PJRT transfer copied every byte twice and held - /// the DMA pumps, which on four B70 made Llama-3.1-8B take 2.0 s - /// into pinned_host against 0.67 s into device memory. - const Kind = union(enum) { - /// Read into pinned blocks in file order, where its transfers - /// address it, then copied to the devices by DMA. - dma_read: Read, - /// Reads its extents, whose items land their bytes in its - /// blocks: a read of its plan's file grouped by destination - /// box (`Planner.JobCut.stage`), or a gathered destination's - /// range (`Planner.JobCut.appendGatherJobs`). - dma_land: Land, - /// Reads nothing: every transfer goes from the loader's zero - /// block (`Planner.JobCut.appendFillTransfers`). - dma_fill, - /// Read straight into host-resident destinations: the read's - /// iovecs are the destination ranges, so the read is the only - /// copy. Needs a read without alignment padding whose - /// transfers address each of its bytes once, in order - /// (`Planner.JobCut.coversReadInOrder`). - host_direct: Read, - /// Read into the request's slot of blocks, then copied into - /// host-resident destinations by the worker: reads widened - /// for direct I/O, whose buffers must be aligned, and - /// overlapping parts. - host_bounce: Read, - /// Writes zeros in place, reading nothing. - host_fill, - }; - - /// `len` bytes from the job's `file_offset`, of which the first - /// `minimum_len` hold tensor data and must exist. The rest is - /// alignment padding the end of the file may cut. - const Read = struct { - len: usize, - minimum_len: usize, - }; - - /// A landing job: `len` bytes in its blocks, landed by the - /// `extent_len` extents of the plan's `extents` from - /// `extent_start`, in order. A gathered range no part covers - /// has none. - const Land = struct { - len: usize, - extent_start: usize, - extent_len: usize, - /// Its blocks start zeroed: bytes no part covers. - zeroed: bool = false, - }; - - /// The blocks of its credit's slot the job reads into or lands - /// in; none for a direct host job or a fill job. - fn blockLen(self: Job, block_size: usize) usize { - return switch (self.kind) { - .dma_read, .host_bounce => |read| Planner.blockCount(read.len, block_size), - .dma_land => |land| Planner.blockCount(land.len, block_size), - .dma_fill, .host_direct, .host_fill => 0, - }; - } - }; - - /// Bytes an extent of a landing job places in its blocks: the runs - /// `first` to `end` of `lattice`, its source offsets in the file - /// and its destination offsets logical, clipped to the logical - /// range `[clip_start, clip_end)`. A byte at logical offset `l`, - /// which `grid.locate(l)` puts at offset `o` of box `b` of the - /// destination's grid, lands at `landings[landing_start + b] +% o` - /// of the job's blocks taken end to end (`walkItem`). - const Item = struct { - destination: *Destination, - lattice: backend.Lattice, - first: u64 = 0, - end: u64 = 1, - clip_start: u64, - clip_end: u64, - landing_start: usize, - }; - - /// One read of a landing job: `len` bytes from `file_offset` of - /// `source_slot`'s file, of which the first `minimum_len` exist, - /// landed by the `item_len` items of the plan's `items` from - /// `item_start`. - const Extent = struct { - source_slot: *SourceSlot, - file_offset: u64, - len: usize, - minimum_len: usize, - /// Read straight into the job's blocks: an exact read whose - /// items' runs tile it, in order. Otherwise read into the - /// worker's bounce buffer and copied from there. - scatter: bool, - item_start: usize, - item_len: usize, - }; - - /// `len` bytes at `landing` of the job's blocks taken end to end, - /// for every device of `writer_mask` at `destination_offset` of its - /// shard. Every transfer stays inside one block, which the worker or - /// `Dma.enqueue` finds from `landing` (`blockPiece`); a direct host - /// job reads at the same landings without blocks. A fill one lands - /// at 0: a DMA one reads the zero block, a host one writes zeros. - const Transfer = struct { - destination: *Destination, - landing: usize, - writer_mask: u64, - destination_offset: usize, - len: usize, - }; - - /// The submission this plan belongs to; set when it is published, - /// under the scheduler mutex, with its completion units. - batch: *Batch = undefined, - /// The file every job of the plan reads. - source_slot: *SourceSlot, - jobs: []Job, - transfers: []Transfer, - /// The landing jobs' extents, their items and the items' landing - /// tables (one entry per box of the item's destination). - extents: []Extent = &.{}, - items: []Item = &.{}, - landings: []u64 = &.{}, - requests: []ReadRequest, - events: []Dma.EventContext, - /// Event slots handed out so far, one `fetchAdd` per submission from - /// any device's pump. - events_used: std.atomic.Value(usize) = .init(0), - /// The tensor bytes the jobs read. - source_bytes: u64, - /// How long the plan took to cut. - planning_ns: u64 = 0, - /// Next job to claim; owned by the scheduler mutex. - cursor: usize = 0, - - fn transfersOf(self: *const Plan, job: Job) []const Transfer { - return self.transfers[job.transfer_start..][0..job.transfer_len]; - } -}; - -/// Builds immutable per-file source jobs and transfer records. Their claim -/// order (file order) is decided here once. -/// For DeepSeek-V4-Flash (148.65 GiB, 69,187 tensors), tensor-local reads made -/// about 69,445 source calls despite a 16 MiB request limit. Coalescing reduced -/// that to 9,524. A rigid grid still produced 78,665 DMA pieces; tensor-safe -/// cuts kept the same read count with 69,572 pieces. Per-tensor device buffers -/// remain intentional: reducing submissions below roughly tensors x destinations would require -/// packed device allocations or device-side scattering, not another read cut. -const Planner = struct { - /// What every plan of a loader is cut by. - const Config = struct { - block_size: usize, - /// The longest read, a whole number of blocks - /// (`backend.requestSize`). - request_size: usize, - /// The one-read limit of a strided part (`stridedRead`, which caps - /// it at `request_size`): the request of the smallest block - /// calibration may pick, so a refusal does not follow its pick. - gather_limit: usize = std.math.maxInt(usize), - /// Submissions a DMA job must save to be staged (`JobCut.stage`): - /// one device's in-flight pieces. - minimum_staging_gain: usize = max_dma_pieces_per_device, - }; - - /// How each file is read: aligned when it is local and the VFS that - /// opens it reads it directly under `policy`, or as `predict` answered - /// for it, decided once per file and kept in its slot. A remote file is - /// read exact without being opened here. Without a VFS (`io` is none, or - /// no alignment) every file takes `alignment`. A gather job reads files - /// other than its plan's, so it asks here for each. `predict` tells, - /// before any file is planned, how the checkpoint's files will read. - /// Widening buffered reads is not free: a widened read served from the - /// page cache costs the DMA sources their block alignment for nothing, - /// and warm replicated Llama on four GB300 lost 3-6% when its first - /// tensor no longer landed at block offset zero. So a file asks the VFS - /// actually owning its handle (`VFS.fromIo`, in `init`), both to avoid - /// that cost and to avoid indexing another VFS's handle table; an `io` - /// that is no VFS never reads directly. - const Reads = struct { - io: std.Io, - vfs: ?*VFS, - policy: VFS.DirectIo, - /// Every read of a file read directly starts and ends at a multiple - /// of this; 0 reads every file exact. - alignment: usize, - /// The files `predict` answered for, by real path as the registry - /// names them (`safetensors.Tensor.file_uri`), each with the policy - /// that decides it as answered: under `auto`, `on` for a file that - /// reads direct and `off` for one that reads buffered. Under `on`, - /// which samples nothing, every file keeps `on`, and a filesystem - /// refusing the flag is logged when the file is planned. Owned until - /// `deinit`. - predicted: std.StringHashMapUnmanaged(VFS.DirectIo) = .empty, - - fn init(io: std.Io, policy: VFS.DirectIo) Reads { - const alignment = backend.readAlignment(io, policy); - return .{ .io = io, .vfs = if (alignment != 0) VFS.fromIo(io) else null, .policy = policy, .alignment = alignment }; - } - - fn deinit(self: *Reads, allocator: std.mem.Allocator) void { - var paths = self.predicted.keyIterator(); - while (paths.next()) |path| allocator.free(path.*); - self.predicted.deinit(allocator); - } - - /// The file's alignment, decided the first time it is asked: a - /// local file is opened here, and decided as `predict` answered for - /// it, else under `policy`. One that fails to open is read exact: - /// its reads fail with the same error (`LazyOnce`), which `readSlot` - /// logs with the path. - fn alignmentOf(self: Reads, slot: *SourceSlot) error{Canceled}!usize { - const vfs = self.vfs orelse return self.alignment; - if (slot.alignment) |known| return known; - // A predicted file is decided as predicted, not sampled again: - // the prediction's sample queued the read of each page it missed - // and of the kernel's readahead, and a second sample of those - // pages once the reads have landed finds the file cached - // (`VFS.predictDirectIo`). Under `on`, a filesystem refusing the - // flag still reads it buffered. - const policy = self.predicted.get(slot.uri) orelse self.policy; - const direct = vfs.isLocal(slot.uri) and vfs.useDirectIo(slot.ensure(self.io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => return 0, - }, policy); - const alignment = if (direct) self.alignment else 0; - slot.alignment = alignment; - return alignment; - } - - /// The widest alignment `alignmentOf` can give the file, without - /// opening it: 0 for a remote file. - fn boundOf(self: Reads, uri: []const u8) usize { - const vfs = self.vfs orelse return self.alignment; - return if (vfs.isLocal(uri)) self.alignment else 0; - } - - /// How the checkpoint's local safetensors files will read, asked - /// once, before any is planned, to place the arenas: each as - /// `VFS.predictDirectIo` answers under `policy`, an answer kept in - /// `predicted` for `alignmentOf` to decide the file by. The files - /// are those `TensorRegistry.fromPath` reads for the checkpoint - /// (`safetensors.checkpointFiles`, which reads its index). Nothing - /// without a VFS, for a checkpoint that is not local, or for files - /// that do not resolve or open: the registry reports those. Keeps - /// no file open, and logs nothing. A wrong prediction (residency - /// changing before the file is planned) costs speed, never bytes. - fn predict(self: *Reads, allocator: std.mem.Allocator, checkpoint: ?[]const u8) error{ Canceled, OutOfMemory }!Prediction { - std.debug.assert(self.predicted.count() == 0); - const vfs = self.vfs orelse return .{}; - const path = checkpoint orelse return .{}; - if (!vfs.isLocal(path)) return .{}; - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const dir, const names = safetensors.checkpointFiles(arena.allocator(), self.io, path) catch |err| switch (err) { - error.Canceled => return error.Canceled, - error.OutOfMemory => return error.OutOfMemory, - else => return .{}, - }; - defer dir.close(self.io); - var prediction: Prediction = .{}; - var real_path_buffer: [std.fs.max_path_bytes]u8 = undefined; - for (names) |name| { - const real_path_len = dir.realPathFile(self.io, name, &real_path_buffer) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => continue, - }; - const real_path = real_path_buffer[0..real_path_len]; - // A symlink and the file it leads to are sampled once. - if (self.predicted.contains(real_path)) continue; - const file = try vfs.predictDirectIo(dir, name, self.policy) orelse continue; - const owned = try allocator.dupe(u8, real_path); - errdefer allocator.free(owned); - try self.predicted.putNoClobber(allocator, owned, if (file.direct or self.policy == .on) .on else .off); - prediction.add(file); - } - return prediction; - } - - /// The bytes of a checkpoint's local safetensors files, and of those - /// that will read direct (`predict`). - const Prediction = struct { - files: usize = 0, - bytes: u64 = 0, - direct_bytes: u64 = 0, - - fn add(self: *Prediction, file: VFS.DirectIoPrediction) void { - self.files += 1; - self.bytes += file.size; - if (file.direct) self.direct_bytes += file.size; - } - - /// Whether the arenas are placed for direct reads - /// (`host_memory.Workspace.init`): files holding at least four - /// fifths of the bytes read direct. In measurements on two GB300 - /// of one Grace node loading Qwen3.8-27B-FP8 (28.75 GiB), - /// spreading the arenas over both nodes saved direct reads about - /// 4.2 ms per GiB forced direct and 4.0 cold under auto, and cost - /// reads through the page cache 10 to 16 ms per GiB: it pays from - /// about 0.71 to 0.80 of the bytes read direct, and four fifths, - /// at the top of that range, leans to the devices' nodes, whose - /// mistake costs less per GiB. - fn direct(self: Prediction) bool { - return self.bytes != 0 and self.direct_bytes * 5 >= self.bytes * 4; - } - }; - }; - - /// Plans `parts` one file at a time and publishes each plan as soon as - /// it exists, so workers claim the first file while the rest is planned, - /// each plan in the batch's arena. Stops at the first error; the caller - /// seals or fails the batch. `reads` says how each file is read - /// (`Reads`). - /// Whole-model planning delayed the first read by 0.32 s for DeepSeek's - /// 46 files on one MI300X; publishing per file overlaps the rest with IO. - fn publishFiles( - scheduler: *Scheduler, - io: std.Io, - batch: *Batch, - parts: []Part, - config: Config, - reads: Reads, - ) !void { - const order = try sortedPartOrder(scheduler.allocator, parts); - defer scheduler.allocator.free(order); - var file_start: usize = 0; - while (file_start < order.len) { - const file_end = fileGroupEnd(parts, order, file_start); - const planning_started: std.Io.Timestamp = .now(io, .awake); - const plan = try preparePlan(scheduler.allocator, batch.arena.allocator(), parts, order[file_start..file_end], config, reads); - plan.planning_ns = @intCast(@max(planning_started.untilNow(io, .awake).nanoseconds, 0)); - try scheduler.publish(io, batch, plan); - if (batch.published_at == null) batch.published_at = .now(io, .awake); - file_start = file_end; - } - } - - /// Plans one file: `order` indexes its `parts` by delivery class, then - /// offset and size (`sortedPartOrder`). `JobCut.cutRuns` cuts the DMA - /// and the host parts at the file's alignment (`Reads.alignmentOf`), - /// `JobCut.appendGatherJobs` each gathered destination with the file of - /// its first part; the jobs merge in file order, then - /// `JobCut.appendFillJob` appends each class's zero fill. The plan goes - /// in `arena` (`JobCut.finish`), the cut in `allocator`. - fn preparePlan( - allocator: std.mem.Allocator, - arena: std.mem.Allocator, - parts: []Part, - order: []const usize, - config: Config, - reads: Reads, - ) !*Plan { - const source_slot = parts[order[0]].source_slot; - for (order) |part_index| std.debug.assert(std.mem.eql(u8, parts[part_index].source.file_uri, parts[order[0]].source.file_uri)); - const dma_end = deliveryEnd(parts, order, 0, .dma); - const host_end = deliveryEnd(parts, order, dma_end, .host); - const alignment = try reads.alignmentOf(source_slot); - var cut: JobCut = .{ - .allocator = allocator, - .parts = parts, - .source_slot = source_slot, - .config = config, - .reads = reads, - .alignment = alignment, - .maximum_job_len = maximumJobLen(config.request_size, alignment), - // A gather job's extents may be in files read directly while - // this one is not, so its ranges leave room for the widest - // alignment. - .gather_len = maximumJobLen(config.request_size, reads.alignment), - }; - defer cut.deinit(); - try cut.cutRuns(order[0..dma_end], false); - try cut.cutRuns(order[dma_end..host_end], true); - for (order[host_end..]) |part_index| { - const part = &parts[part_index]; - std.debug.assert(part.destination.gathered); - if (part == &part.destination.parts[0]) try cut.appendGatherJobs(part.destination); - } - // Jobs are claimed in planning order, which is file order (a - // gather job by its first read). A byte-fair order across - // destination devices was measured against it on the two hosts - // where the DMA pump is not the ceiling and made no difference. - // Each class is in file order already; the stable sort interleaves - // them. - std.mem.sort(Plan.Job, cut.jobs.items, {}, struct { - fn lessThan(_: void, lhs: Plan.Job, rhs: Plan.Job) bool { - return lhs.file_offset < rhs.file_offset; - } - }.lessThan); - - // The zero fill of each destination goes out once, with the plan - // of its first part; it depends on no read, so one job after this - // file's reads carries all of it, one per delivery class. - try cut.appendFillJob(order[0..dma_end], false); - try cut.appendFillJob(order[dma_end..host_end], true); - return cut.finish(arena); - } - - /// The end of the parts of `delivery` in `order` from `start` on. - fn deliveryEnd(parts: []const Part, order: []const usize, start: usize, delivery: Destination.Delivery) usize { - var end = start; - while (end < order.len and parts[order[end]].destination.delivery() == delivery) : (end += 1) {} - return end; - } - - /// The jobs and transfers of one plan as `preparePlan` cuts them. - const JobCut = struct { - /// One part's bytes in a read job: the logical range `[start, end)` - /// of its destination, from `file_offset` of the file on. - const Segment = struct { destination: *Destination, start: u64, end: u64, file_offset: u64 }; - /// A segment's bytes on box `box` of its destination: `len` from - /// `box_offset`, and where a staged job lands them. - const Entry = struct { segment: u32, box: u32, group: u32 = 0, box_offset: u64, len: u64, landing: u64 = 0 }; - /// What a staged job lands together. - const Group = struct { destination: *Destination, box: u32 }; - - allocator: std.mem.Allocator, - parts: []Part, - /// The file every job of the plan reads. - source_slot: *SourceSlot, - config: Config, - reads: Reads, - /// The alignment of the plan's file (`Reads.alignmentOf`). - alignment: usize, - maximum_job_len: usize, - /// The longest logical range of a gather job. - gather_len: usize, - jobs: std.ArrayList(Plan.Job) = .empty, - transfers: std.ArrayList(Plan.Transfer) = .empty, - items: std.ArrayList(Plan.Item) = .empty, - landings: std.ArrayList(u64) = .empty, - extents: std.ArrayList(Plan.Extent) = .empty, - safe_boundaries: std.ArrayList(u64) = .empty, - /// A read job's segments before they are placed, and the staging - /// scratch. - segments: std.ArrayList(Segment) = .empty, - entries: std.ArrayList(Entry) = .empty, - groups: std.AutoHashMapUnmanaged(Group, u32) = .empty, - source_bytes: u64 = 0, - - /// The plan of the jobs cut, copied into `arena` with every context - /// its jobs can need, allocated before publication makes it visible - /// to workers: a request per job and an event per writer of a DMA - /// job's transfer. - fn finish(self: *JobCut, arena: std.mem.Allocator) !*Plan { - const jobs = self.jobs.items; - const transfers = self.transfers.items; - var dma_submissions: usize = 0; - for (jobs) |job| switch (job.kind) { - .dma_read, .dma_land, .dma_fill => dma_submissions += submissionCount(transfers[job.transfer_start..][0..job.transfer_len]), - .host_direct, .host_bounce, .host_fill => {}, - }; - const requests = try arena.alloc(ReadRequest, jobs.len); - @memset(requests, ReadRequest.idle); - const plan = try arena.create(Plan); - plan.* = .{ - .source_slot = self.source_slot, - .jobs = try arena.dupe(Plan.Job, jobs), - .transfers = try arena.dupe(Plan.Transfer, transfers), - .extents = try arena.dupe(Plan.Extent, self.extents.items), - .items = try arena.dupe(Plan.Item, self.items.items), - .landings = try arena.dupe(u64, self.landings.items), - .requests = requests, - .events = try arena.alloc(Dma.EventContext, dma_submissions), - .source_bytes = self.source_bytes, - }; - return plan; - } - - fn deinit(self: *JobCut) void { - const allocator = self.allocator; - self.groups.deinit(allocator); - self.entries.deinit(allocator); - self.segments.deinit(allocator); - self.safe_boundaries.deinit(allocator); - self.extents.deinit(allocator); - self.landings.deinit(allocator); - self.items.deinit(allocator); - self.transfers.deinit(allocator); - self.jobs.deinit(allocator); - } - - /// Cuts the runs of `order` (part indices of one delivery class in - /// file order) into jobs: each run of touching or overlapping - /// ranges into the minimum number of jobs, cut at part-safe - /// boundaries where that number allows. No job depends on another, - /// since every DMA piece is submitted as soon as its block is read. - /// With an alignment (`Reads.alignmentOf`), a job's read widens to - /// aligned bounds around its range (`maximumJobLen` leaves room for - /// it), and the transfers address the widened read. A DMA job's - /// segments are placed in its blocks by `placeDma`. A host job is - /// `host_direct` only when its read is exact and its transfers - /// cover it in order (`coversReadInOrder`), otherwise `host_bounce`. - fn cutRuns(self: *JobCut, order: []const usize, host: bool) !void { - const allocator = self.allocator; - const parts = self.parts; - const alignment = self.alignment; - const maximum_job_len = self.maximum_job_len; - var run_cursor: usize = 0; - while (run_cursor < order.len) { - self.safe_boundaries.clearRetainingCapacity(); - const first_range = parts[order[run_cursor]].fileRange(); - std.debug.assert(first_range.end > first_range.start); // backend.LoadSpec: parts are non-empty - const first_offset = first_range.start; - var run_end = first_range.end; - var run_part_end = run_cursor + 1; - while (run_part_end < order.len) : (run_part_end += 1) { - const candidate = parts[order[run_part_end]].fileRange(); - if (candidate.start > run_end) break; - // A touching range starts where every preceding range has - // ended. Unlike an arbitrary part end, this is safe even - // when the batch contains overlapping or duplicate ranges. - if (candidate.start == run_end and - (self.safe_boundaries.items.len == 0 or - self.safe_boundaries.items[self.safe_boundaries.items.len - 1] != candidate.start)) - { - try self.safe_boundaries.append(allocator, candidate.start); - } - run_end = @max(run_end, candidate.end); - } - - var job_start = first_offset; - var candidate_start = run_cursor; - var boundary_cursor: usize = 0; - const run_len = run_end - first_offset; - const maximum_job_len_u64: u64 = @intCast(maximum_job_len); - var jobs_remaining: usize = @intCast(run_len / maximum_job_len_u64 + - @intFromBool(run_len % maximum_job_len_u64 != 0)); - while (jobs_remaining != 0) { - const safe_boundaries = self.safe_boundaries.items; - const hard_end = @min( - run_end, - std.math.add(u64, job_start, maximum_job_len) catch run_end, - ); - const job_end = if (jobs_remaining == 1) - run_end - else boundary: { - // Less than the run's length, which is a u64. - const remaining_capacity = @as(u64, @intCast(jobs_remaining - 1)) * maximum_job_len_u64; - const minimum_end = @max(job_start + 1, run_end - remaining_capacity); - const maximum_end = @min( - hard_end, - run_end - @as(u64, @intCast(jobs_remaining - 1)), - ); - while (boundary_cursor < safe_boundaries.len and - safe_boundaries[boundary_cursor] <= job_start) - { - boundary_cursor += 1; - } - var scan = boundary_cursor; - var selected: ?u64 = null; - while (scan < safe_boundaries.len and - safe_boundaries[scan] <= maximum_end) : (scan += 1) - { - if (safe_boundaries[scan] >= minimum_end) - selected = safe_boundaries[scan]; - } - boundary_cursor = scan; - break :boundary selected orelse maximum_end; - }; - const job_len: usize = @intCast(job_end - job_start); - const read_start = if (alignment == 0) job_start else std.mem.alignBackward(u64, job_start, alignment); - const read_end = if (alignment == 0) job_end else std.mem.alignForward(u64, job_end, alignment); - const read_len: usize = @intCast(read_end - read_start); - const transfer_start = self.transfers.items.len; - // The job's segments first: where they land in its - // blocks depends on all of them. - self.segments.clearRetainingCapacity(); - while (candidate_start < run_part_end) { - const candidate = parts[order[candidate_start]].fileRange(); - if (candidate.end > job_start) break; - candidate_start += 1; - } - for (order[candidate_start..run_part_end]) |part_index| { - const part = &parts[part_index]; - const range = part.fileRange(); - if (range.start >= job_end) break; - const intersection_start = @max(job_start, range.start); - const intersection_end = @min(job_end, range.end); - if (intersection_start >= intersection_end) continue; - const logical_start = part.lattice.dst_offset + (intersection_start - range.start); - try self.segments.append(allocator, .{ - .destination = part.destination, - .start = logical_start, - .end = logical_start + (intersection_end - intersection_start), - .file_offset = intersection_start, - }); - } - std.debug.assert(self.segments.items.len != 0); - const read: Plan.Job.Read = .{ .len = read_len, .minimum_len = @intCast(job_end - read_start) }; - const kind: Plan.Job.Kind = if (host) host: { - for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); - break :host if (alignment == 0 and coversReadInOrder(self.transfers.items[transfer_start..], read_len)) - .{ .host_direct = read } - else - .{ .host_bounce = read }; - } else try self.placeDma(read, read_start, job_start, job_len); - try self.jobs.append(allocator, .{ - .file_offset = read_start, - .transfer_start = transfer_start, - .transfer_len = self.transfers.items.len - transfer_start, - .kind = kind, - }); - self.source_bytes +|= @intCast(job_len); - job_start = job_end; - jobs_remaining -= 1; - } - std.debug.assert(job_start == run_end); - run_cursor = run_part_end; - } - } - - /// Places a DMA job's segments (`segments`, in file order) in its - /// blocks. Blocks holding the read in file order give one transfer - /// per run (`ShardGrid.run`): a tensor sharded along any axis but - /// its first takes one per row and device, as a column-sharded - /// `o_proj` or `down_proj` does. A staged job lands its read grouped - /// by destination box instead (`stage`), so a box takes one piece - /// per block. Llama-3.1-8B sharded like that on four B70, page - /// cache warm, planned 1.05M submissions and loaded in 5.1 s (~1.2 - /// ms per piece, 64 in flight per device); staged, 5,468 and 0.71 - /// s, against 0.67 s sharded by rows. Staging costs a read - /// scattered over the runs, or for a direct read a copy through the - /// worker's bounce buffer (`landJob`), so a job is staged only where - /// it saves `Config.minimum_staging_gain` submissions, and only when - /// its segments tile its bytes once: overlapping parts send a source - /// byte to two segments, and one read cannot land it twice. - fn placeDma(self: *JobCut, read: Plan.Job.Read, read_start: u64, data_start: u64, job_len: usize) !Plan.Job.Kind { - const block_size = self.config.block_size; - const transfer_start = self.transfers.items.len; - // File order takes a submission per writer of every run, cut by - // blocks, less where runs of adjacent segments continue each other - // or a run ends where a block does: at most `slack` fewer. - var in_file_order: usize = 0; - var slack: usize = 0; - var tiles = true; - var next = data_start; - for (self.segments.items) |segment| { - if (segment.file_offset != next) tiles = false; - next = segment.file_offset + (segment.end - segment.start); - const grid = &segment.destination.grid; - const landing = segment.file_offset - read_start; - const crossings = (landing + segment.end - segment.start - 1) / block_size - landing / block_size; - in_file_order += (grid.runCount(segment.start, segment.end) + crossings) * grid.replicas(); - slack += (1 + crossings) * grid.replicas(); - } - if (tiles and next == data_start + job_len) { - if (try self.stage(read, read_start, job_len, in_file_order, slack)) |land| return .{ .dma_land = land }; - } - for (self.segments.items) |segment| try self.appendRunTransfers(transfer_start, segment, read_start); - return .{ .dma_read = read }; - } - - /// Lands the job's segments box by box: their bytes on each box of a - /// destination, grouped by (destination, box) in order of first - /// appearance, each group in box order. The job's read is its one - /// extent, and one `Item` per segment tells the worker where its bytes - /// go (`landJob`). Null, with nothing appended, when that saves - /// fewer than `Config.minimum_staging_gain` submissions against file - /// order, estimated at `in_file_order` and at most `slack` fewer: - /// counted exactly, by listing the runs, only when that decides. - fn stage(self: *JobCut, read: Plan.Job.Read, read_start: u64, job_len: usize, in_file_order: usize, slack: usize) !?Plan.Job.Land { - const allocator = self.allocator; - const segments = self.segments.items; - self.entries.clearRetainingCapacity(); - self.groups.clearRetainingCapacity(); - for (segments, 0..) |segment, index| { - const grid = &segment.destination.grid; - for (0..grid.box_count) |box| { - const low = grid.rankIn(box, segment.start); - const high = grid.rankIn(box, segment.end); - if (high == low) continue; - const group = try self.groups.getOrPut(allocator, .{ .destination = segment.destination, .box = @intCast(box) }); - if (!group.found_existing) group.value_ptr.* = @intCast(self.groups.count() - 1); - try self.entries.append(allocator, .{ .segment = @intCast(index), .box = @intCast(box), .group = group.value_ptr.*, .box_offset = low, .len = high - low }); - } - } - const entries = self.entries.items; - std.mem.sort(Entry, entries, {}, struct { - fn lessThan(_: void, lhs: Entry, rhs: Entry) bool { - if (lhs.group != rhs.group) return lhs.group < rhs.group; - if (lhs.box_offset != rhs.box_offset) return lhs.box_offset < rhs.box_offset; - return lhs.segment < rhs.segment; - } - }.lessThan); - // The read's own order when every box of a segment is one run and - // the groups come in the order of the runs. - var runs: u64 = 0; - for (segments) |segment| runs += segment.destination.grid.runCount(segment.start, segment.end); - if (runs == entries.len) { - const in_order = for (entries[1..], entries[0 .. entries.len - 1]) |entry, previous| { - if (entry.segment < previous.segment or (entry.segment == previous.segment and entry.box < previous.box)) break false; - } else true; - if (in_order) return null; - } - const staged_start = self.transfers.items.len; - var landing: u64 = 0; - for (entries) |*entry| { - const destination = segments[entry.segment].destination; - entry.landing = landing; - try self.appendBlocked(staged_start, .{ - .destination = destination, - .landing = @intCast(landing), - .writer_mask = destination.grid.writers[entry.box], - .destination_offset = @intCast(entry.box_offset), - .len = @intCast(entry.len), - }); - landing += entry.len; - } - std.debug.assert(landing == job_len); - const needed = submissionCount(self.transfers.items[staged_start..]) +| self.config.minimum_staging_gain; - var file_order_wins = needed > in_file_order; - if (!file_order_wins and needed +| slack > in_file_order) { - const exact_start = self.transfers.items.len; - for (segments) |segment| try self.appendRunTransfers(exact_start, segment, read_start); - file_order_wins = needed > submissionCount(self.transfers.items[exact_start..]); - self.transfers.shrinkRetainingCapacity(exact_start); - } - if (file_order_wins) { - self.transfers.shrinkRetainingCapacity(staged_start); - return null; - } - - // A landing table per segment, one entry per box of its - // destination. - const item_start = self.items.items.len; - for (segments) |segment| { - const landing_start = self.landings.items.len; - try self.landings.appendNTimes(allocator, 0, segment.destination.grid.box_count); - try self.items.append(allocator, .{ - .destination = segment.destination, - .lattice = .{ .src_offset = segment.file_offset, .dst_offset = segment.start, .width = segment.end - segment.start }, - .clip_start = segment.start, - .clip_end = segment.end, - .landing_start = landing_start, - }); - } - for (entries) |entry| { - const item = self.items.items[item_start + entry.segment]; - self.landings.items[item.landing_start + entry.box] = entry.landing -% entry.box_offset; - } - // Only a direct read has padding around its tensor bytes, which - // goes through the worker's bounce buffer. - const extent_start = self.extents.items.len; - try self.extents.append(allocator, .{ - .source_slot = self.source_slot, - .file_offset = read_start, - .len = read.len, - .minimum_len = read.minimum_len, - .scatter = self.alignment == 0, - .item_start = item_start, - .item_len = segments.len, - }); - return .{ .len = job_len, .extent_start = extent_start, .extent_len = 1 }; - } - - /// Appends `segment`'s transfers with its bytes where the read holds - /// them, one per run (`ShardGrid.run`) and block, merged with the - /// previous transfer from `transfer_start` on where block, writers - /// and both offsets continue it: parts adjacent in the file and in - /// the destination go out as one transfer. - fn appendRunTransfers(self: *JobCut, transfer_start: usize, segment: Segment, read_start: u64) !void { - const destination = segment.destination; - const grid = &destination.grid; - var at = segment.start; - while (at < segment.end) { - const stop = grid.runEnd(at, segment.end); - const box, const box_offset = grid.locate(at); - try self.appendBlocked(transfer_start, .{ - .destination = destination, - .landing = @intCast(segment.file_offset + (at - segment.start) - read_start), - .writer_mask = grid.writers[box], - .destination_offset = @intCast(box_offset), - .len = @intCast(stop - at), - }); - at = stop; - } - } - - /// Plans a gathered destination (`Destination.gathered`) whole: its - /// logical bytes cut into ranges of at most `gather_len`, each one - /// job. For every part with runs in the range the job reads the - /// source range those runs come from (an `Extent`, from whichever - /// file holds it), and it lands the range in its blocks box by box, - /// one transfer per box and block, its bytes no part covers zero. - /// A part whose source runs are out of order (a broadcast repeats - /// them, a transpose permutes them) reads its whole source range - /// in every job (`stridedRead`). An interleave of members reads - /// each member's rows of the range, so the range lands in one piece - /// per box and block whatever the interleave's granularity. - fn appendGatherJobs(self: *JobCut, destination: *Destination) !void { - const allocator = self.allocator; - const grid = &destination.grid; - const parts = destination.parts; - // reach[i]: the furthest destination byte parts[0..=i] reach, so - // a range's first candidate is found by bisection. - const reach = try allocator.alloc(u64, parts.len); - defer allocator.free(reach); - var furthest: u64 = 0; - for (parts, reach) |*part, *at| { - const spans = part.lattice.spans().?; - furthest = @max(furthest, part.lattice.dst_offset + spans.dst); - at.* = furthest; - } - var start: u64 = 0; - while (start < grid.total) { - const end = @min(grid.total, start + self.gather_len); - const transfer_start = self.transfers.items.len; - const extent_start = self.extents.items.len; - const landing_start = self.landings.items.len; - try self.landings.appendNTimes(allocator, 0, grid.box_count); - var landing: u64 = 0; - for (0..grid.box_count) |box| { - const low = grid.rankIn(box, start); - const high = grid.rankIn(box, end); - if (high == low) continue; - self.landings.items[landing_start + box] = landing -% low; - try self.appendBlocked(transfer_start, .{ - .destination = destination, - .landing = @intCast(landing), - .writer_mask = grid.writers[box], - .destination_offset = @intCast(low), - .len = @intCast(high - low), - }); - landing += high - low; - } - std.debug.assert(landing == end - start); - - var covered: u64 = 0; - var index = std.sort.partitionPoint(u64, reach, start, struct { - fn below(limit: u64, at: u64) bool { - return at <= limit; - } - }.below); - while (index < parts.len and parts[index].lattice.dst_offset < end) : (index += 1) { - const part = &parts[index]; - const first = part.lattice.firstEndingAfter(start); - const last = part.lattice.firstStartingFrom(end); - if (first >= last) continue; - covered += part.lattice.coveredIn(start, end); - try self.appendExtent(part, first, last, start, end, landing_start); - } - std.debug.assert(covered <= end - start); - const job_len: usize = @intCast(end - start); - const extent_len = self.extents.items.len - extent_start; - try self.jobs.append(allocator, .{ - .file_offset = if (extent_len != 0) self.extents.items[extent_start].file_offset else 0, - .transfer_start = transfer_start, - .transfer_len = self.transfers.items.len - transfer_start, - .kind = .{ .dma_land = .{ - .len = job_len, - .extent_start = extent_start, - .extent_len = extent_len, - .zeroed = covered < end - start, - } }, - }); - start = end; - } - } - - /// The extent reading `part`'s runs `first` to `last` (exclusive) - /// clipped to `[start, end)`, with the one item that places them: - /// from the first run's first byte in the range to the last run's - /// last one when the part's source runs are in order, else the - /// part's whole source range. - fn appendExtent(self: *JobCut, part: *const Part, first: u64, last: u64, start: u64, end: u64, landing_start: usize) !void { - const lattice = part.lattice; - const width = lattice.width; - var source_start = lattice.src_offset; - var source_end = lattice.src_offset + lattice.spans().?.src; - const ordered = lattice.srcOrdered(); - if (ordered) { - const head = lattice.at(first); - const tail = lattice.at(last - 1); - source_start = head.src + (start -| head.dst); - source_end = tail.src + width - ((tail.dst + width) -| end); - } - const file_start = part.source.offset + source_start; - const file_end = part.source.offset + source_end; - const alignment = try self.reads.alignmentOf(part.source_slot); - const read_start = if (alignment == 0) file_start else std.mem.alignBackward(u64, file_start, alignment); - const read_end = if (alignment == 0) file_end else std.mem.alignForward(u64, file_end, alignment); - // The range of a part whose runs are back to back is the job's; - // any other part's whole range fits (`stridedRead`). - std.debug.assert(read_end - read_start <= self.config.request_size); - try self.extents.append(self.allocator, .{ - .source_slot = part.source_slot, - .file_offset = read_start, - .len = @intCast(read_end - read_start), - .minimum_len = @intCast(file_end - read_start), - .scatter = alignment == 0 and ordered and lattice.srcTiled(), - .item_start = self.items.items.len, - .item_len = 1, - }); - try self.items.append(self.allocator, .{ - .destination = part.destination, - .lattice = .{ .src_offset = part.source.offset + lattice.src_offset, .dst_offset = lattice.dst_offset, .width = width, .dims = lattice.dims }, - .first = first, - .end = last, - .clip_start = start, - .clip_end = end, - .landing_start = landing_start, - }); - self.source_bytes +|= file_end - file_start; - } - - /// Appends `transfer`, cut at block boundaries, merged with the - /// previous transfer from `transfer_start` on where block, writers - /// and both offsets continue it. - fn appendBlocked(self: *JobCut, transfer_start: usize, transfer: Plan.Transfer) !void { - const block_size = self.config.block_size; - var cursor: usize = 0; - while (cursor < transfer.len) { - const at = transfer.landing + cursor; - const in_block = at % block_size; - const take = @min(transfer.len - cursor, block_size - in_block); - const destination_offset = transfer.destination_offset + cursor; - cursor += take; - // A transfer that ends where this one starts is in its - // block unless a block starts here. - if (self.transfers.items.len > transfer_start and in_block != 0) { - const previous = &self.transfers.items[self.transfers.items.len - 1]; - if (previous.destination == transfer.destination and - previous.writer_mask == transfer.writer_mask and - previous.landing + previous.len == at and - previous.destination_offset + previous.len == destination_offset) - { - previous.len += take; - continue; - } - } - try self.transfers.append(self.allocator, .{ - .destination = transfer.destination, - .landing = at, - .writer_mask = transfer.writer_mask, - .destination_offset = destination_offset, - .len = take, - }); - } - } - - fn submissionCount(transfers: []const Plan.Transfer) usize { - var count: usize = 0; - for (transfers) |transfer| count += @popCount(transfer.writer_mask); - return count; - } - - /// The zero fill of the destinations whose first part is among - /// `order`, the file's parts of one delivery class, as one job. A DMA - /// fill job reads one block, the zero block; a host one writes zeros - /// in place. A gathered destination's jobs write its zeros - /// themselves. - fn appendFillJob(self: *JobCut, order: []const usize, host: bool) !void { - const fill_start = self.transfers.items.len; - for (order) |part_index| { - const part = &self.parts[part_index]; - const destination = part.destination; - if (part != &destination.parts[0]) continue; - try self.appendFillTransfers(fill_start, destination); - } - const fill_transfers = self.transfers.items[fill_start..]; - if (fill_transfers.len == 0) return; - try self.jobs.append(self.allocator, .{ - .file_offset = 0, - .transfer_start = fill_start, - .transfer_len = fill_transfers.len, - .kind = if (host) .host_fill else .dma_fill, - }); - } - - /// Appends the zero-fill transfers of `destination`: every logical - /// range no part covers (the gaps between its parts and the tail), - /// placed by its grid like read bytes, each piece at most one block - /// at landing 0, where a DMA job reads the zero block. Pieces count - /// toward their targets' totals, so the fill completes each - /// device's placement bytes alongside the parts. - fn appendFillTransfers(self: *JobCut, transfer_start: usize, destination: *Destination) !void { - var covered_end: u64 = 0; - for (destination.parts) |part| { - const lattice = part.lattice; - if (lattice.dst_offset > covered_end) try self.appendFillRange(transfer_start, destination, covered_end, lattice.dst_offset); - covered_end = lattice.dst_offset + lattice.width; - } - if (covered_end < destination.grid.total) try self.appendFillRange(transfer_start, destination, covered_end, destination.grid.total); - } - - /// The fill of the logical range `[start, end)`: one range of each box - /// it touches, cut by blocks, merged where one writer set's pieces - /// are contiguous. - fn appendFillRange(self: *JobCut, transfer_start: usize, destination: *Destination, start: u64, end: u64) !void { - const block_size = self.config.block_size; - const grid = &destination.grid; - for (0..grid.box_count) |box| { - var destination_offset: usize = @intCast(grid.rankIn(box, start)); - var remaining: usize = @intCast(grid.rankIn(box, end) - destination_offset); - const writer_mask = grid.writers[box]; - while (remaining != 0) { - if (self.transfers.items.len > transfer_start) merge: { - const previous = &self.transfers.items[self.transfers.items.len - 1]; - if (previous.destination != destination or previous.writer_mask != writer_mask or - previous.destination_offset + previous.len != destination_offset or - previous.len == block_size) - break :merge; - const grown = @min(remaining, block_size - previous.len); - previous.len += grown; - destination_offset += grown; - remaining -= grown; - continue; - } - const take = @min(remaining, block_size); - try self.transfers.append(self.allocator, .{ - .destination = destination, - .landing = 0, - .writer_mask = writer_mask, - .destination_offset = destination_offset, - .len = take, - }); - destination_offset += take; - remaining -= take; - } - } - } - - /// Whether a host job's transfers address its read from byte zero - /// to `read_len` in order, each source byte once: then the read - /// can scatter straight into the destinations. Overlapping or - /// duplicate parts address some source bytes twice and go through - /// blocks instead. - fn coversReadInOrder(transfers: []const Plan.Transfer, read_len: usize) bool { - var next: usize = 0; - for (transfers) |transfer| { - if (transfer.landing != next) return false; - next += transfer.len; - } - return next == read_len; - } - }; - fn blockCount(len: usize, block_size: usize) usize { - return len / block_size + @intFromBool(len % block_size != 0); - } - - /// The source bytes every job of `part` reads whole, against one read - /// (`Config.gather_limit`, at most a request). A part whose source runs - /// are back to back reads only its job's range: 0. Any other counts its - /// whole source range, widened to the widest alignment its file may be - /// read at (`Reads.boundOf`): out of order (a broadcast repeats its - /// runs, a transpose permutes them), every job reads that range - /// (`JobCut.appendExtent`); in order with gaps (a slice along an inner - /// axis), a job reads from its first run to its last, which the gaps - /// stretch past any job's length. - fn stridedRead(config: Config, reads: Reads, part: LoadPart) backend.Backend.StridedRead { - const lattice = part.lattice; - const limit = @min(config.request_size, config.gather_limit); - if (lattice.srcTiled()) return .{ .read = 0, .limit = limit }; - const alignment = @max(reads.boundOf(part.source.file_uri), 1); - const start = part.source.offset + lattice.src_offset; - const end = start + lattice.spans().?.src; - return .{ - .read = std.mem.alignForward(u64, end, alignment) - std.mem.alignBackward(u64, start, alignment), - .limit = limit, - }; - } - - /// The longest tensor range one job may cover: the request size, less - /// the two alignment units its widened read can add, so a widened job - /// still fits a request (a credit's slot). `backend.requestSize` refuses - /// a request that cannot hold them. - fn maximumJobLen(request_size: usize, alignment: usize) usize { - std.debug.assert(request_size != 0); - if (alignment == 0) return request_size; - std.debug.assert(std.math.isPowerOfTwo(alignment) and 2 * alignment < request_size); - return request_size - 2 * alignment; - } - - /// Part indices sorted by file URI, delivery class - /// (`Destination.Delivery`), offset, size and index: the planner's - /// input, taken one file group at a time (`fileGroupEnd`), each class - /// in file order. - fn sortedPartOrder(allocator: std.mem.Allocator, parts: []const Part) ![]usize { - const order = try allocator.alloc(usize, parts.len); - for (order, 0..) |*index, i| index.* = i; - const SortContext = struct { - parts: []const Part, - - fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { - const left = ctx.parts[lhs]; - const right = ctx.parts[rhs]; - const uri_order = std.mem.order(u8, left.source.file_uri, right.source.file_uri); - if (uri_order != .eq) return uri_order == .lt; - const left_delivery = left.destination.delivery(); - const right_delivery = right.destination.delivery(); - if (left_delivery != right_delivery) return @backingInt(left_delivery) < @backingInt(right_delivery); - const left_offset = left.fileRange().start; - const right_offset = right.fileRange().start; - if (left_offset != right_offset) return left_offset < right_offset; - if (left.lattice.width != right.lattice.width) return left.lattice.width < right.lattice.width; - return lhs < rhs; - } - }; - std.mem.sort(usize, order, SortContext{ .parts = parts }, SortContext.lessThan); - return order; - } - - /// The end of the file group that begins at `order[start]`. - fn fileGroupEnd(parts: []const Part, order: []const usize, start: usize) usize { - const uri = parts[order[start]].source.file_uri; - var end = start + 1; - while (end < order.len and std.mem.eql(u8, uri, parts[order[end]].source.file_uri)) : (end += 1) {} - return end; - } -}; - -/// A strict FIFO of published plans. A submission holds one plan per source -/// file, published as soon as that file is planned, so a plan reaches the -/// queue behind every earlier plan of its own submission and of every -/// earlier one (submissions publish and seal on one task). Within a plan, -/// jobs are handed out in the planned order (file order); a plan leaves the -/// queue with its last claim or through `fail`. -/// A plan is queued only with jobs left and its completion units are added -/// in the same critical section, so a queued plan's batch still holds at -/// least one unit and can never be freed under the scheduler. -const Scheduler = struct { - /// A claimed job: its position in the plan that owns it. The claim holds - /// one of the batch's completion units until the request releases it. - const Claim = struct { - plan: *Plan, - index: usize, - }; - - allocator: std.mem.Allocator, - /// Plans with unclaimed jobs in publish order. - queue: std.Deque(*Plan) = .empty, - /// Set by `fail`: `publish` refuses plans (`Canceled`) and `waitForWork` - /// returns false. - stopping: bool = false, - mutex: std.Io.Mutex = .init, - condition: std.Io.Condition = .init, - - fn init(allocator: std.mem.Allocator) Scheduler { - return .{ .allocator = allocator }; - } - - fn deinit(self: *Scheduler) void { - // Every batch was awaited (and so claimed out or retired) first. - std.debug.assert(self.queue.len == 0); - self.queue.deinit(self.allocator); - self.* = undefined; - } - - /// Publishes one plan of an open submission behind every earlier plan. - /// A plan without jobs never joins the queue. `Canceled` once the - /// scheduler failed (`fail`). - fn publish(self: *Scheduler, io: std.Io, batch: *Batch, plan: *Plan) !void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - if (self.stopping) return error.Canceled; - const job_count = plan.jobs.len; - try batch.plans.ensureUnusedCapacity(batch.arena.allocator(), 1); - if (job_count != 0) try self.queue.ensureUnusedCapacity(self.allocator, 1); - // Nothing below fails: the plan, its units and the queue entry - // appear together. - plan.batch = batch; - batch.plans.appendAssumeCapacity(plan); - _ = batch.remaining.fetchAdd(job_count, .acq_rel); - if (job_count != 0) { - self.queue.pushBackAssumeCapacity(plan); - self.condition.broadcast(io); - } - } - - /// Stops claims and publishes, wakes every worker waiting for work, and - /// retires the unclaimed units of every queued plan so its batch still - /// reaches `done` through its claimed requests (an open one through its - /// seal and sentinel as well). Claims and this pass both move cursors - /// under the mutex, so they partition the jobs exactly: every claimed - /// job keeps its unit with its worker. Each queued plan still holds an - /// unclaimed unit of its batch, so only the retire of a batch's last - /// queued plan can complete it, and nothing touches the batch after that. - fn fail(self: *Scheduler, io: std.Io) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - self.stopping = true; - self.condition.broadcast(io); - while (self.queue.popFront()) |plan| { - const unclaimed = plan.jobs.len - plan.cursor; - plan.cursor = plan.jobs.len; - plan.batch.releaseUnits(unclaimed); - } - } - - /// Hands out the head plan's next job; the plan leaves the queue with - /// its last one. Null only when nothing is queued: a queued plan always - /// has a job left. - fn claim(self: *Scheduler, io: std.Io) ?Claim { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - const plan = self.queue.front() orelse return null; - const index = plan.cursor; - std.debug.assert(index < plan.jobs.len); - plan.cursor += 1; - if (plan.cursor == plan.jobs.len) _ = self.queue.popFront(); - return .{ .plan = plan, .index = index }; - } - - fn waitForWork(self: *Scheduler, io: std.Io) bool { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (!self.stopping and self.queue.len == 0) { - self.condition.waitUncancelable(io, &self.mutex); - } - return !self.stopping; - } -}; - -/// A claimed source job and its completion state. The plan owns its storage. -const ReadRequest = struct { - const Scratch = struct { - allocator: std.mem.Allocator, - /// A read's ranges: one per block of a job (`readBlocks`), or a - /// scattered read's, at most one vectored read's worth (`Scatter`). - iovecs: [][]u8, - /// A landing job's reads that do not scatter (`landJob`), allocated - /// on first use: as large as a job's blocks, page aligned for - /// O_DIRECT. - bounce: []align(std.heap.page_size_min) u8 = &.{}, - bounce_capacity: usize, - - fn init(allocator: std.mem.Allocator, maximum_blocks: usize, block_size: usize) !Scratch { - return .{ - .allocator = allocator, - .iovecs = try allocator.alloc([]u8, @max(maximum_blocks, max_positional_iovecs)), - .bounce_capacity = maximum_blocks * block_size, - }; - } - - /// The ranges a `Scatter` gathers before it reads. - fn scatterIovecs(self: *const Scratch) [][]u8 { - return self.iovecs[0..max_positional_iovecs]; - } - - fn bounceBuffer(self: *Scratch, len: usize) ![]u8 { - std.debug.assert(len <= self.bounce_capacity); - if (self.bounce.len == 0) self.bounce = try self.allocator.alignedAlloc(u8, .fromByteUnits(std.heap.page_size_min), self.bounce_capacity); - return self.bounce[0..len]; - } - - fn deinit(self: *Scratch) void { - self.allocator.free(self.bounce); - self.allocator.free(self.iovecs); - self.* = undefined; - } - }; - - loader: *Loader, - plan: *Plan, - /// The job's index in its plan, which holds this request at the same - /// index. - index: usize, - /// The request's lifecycle credit: its slot of blocks (`Credits`), which - /// only this request reads into and transfers from until it ends. - slot: u32, - /// The scheduling sentinel and one reference per DMA piece queued; the - /// last release gives the credit back and finishes the job. - pending: std.atomic.Value(usize) = .init(1), - /// Awake-clock nanoseconds of the enqueue; 0 until then. - enqueued_ns: u64 = 0, - - /// The slot of a job that was never claimed: nothing pending, so the - /// retirement checks hold. - const idle: ReadRequest = .{ - .loader = undefined, - .plan = undefined, - .index = undefined, - .slot = undefined, - .pending = .init(0), - }; - - /// Takes the claimed job's request context with the worker's credit. - /// The claim holds the batch's completion unit; the request's final - /// reference drop releases it. - fn init(loader: *Loader, claim: Scheduler.Claim, slot: u32) *ReadRequest { - const self = &claim.plan.requests[claim.index]; - self.* = .{ - .loader = loader, - .plan = claim.plan, - .index = claim.index, - .slot = slot, - }; - return self; - } - - fn run(self: *ReadRequest, scratch: *Scratch) !void { - const loader = self.loader; - const io = loader.io; - const block_size = loader.plan_config.block_size; - const job = self.plan.jobs[self.index]; - const transfers = self.plan.transfersOf(job); - // The first worker to touch a destination creates its state; a - // host-resident one's is normally created ahead by the allocation - // task. - var previous: ?*Destination = null; - for (transfers) |transfer| { - const destination = transfer.destination; - if (destination == previous) continue; - previous = destination; - const init_started = awakeNs(io); - _ = destination.ensureState(loader) catch |err| { - loader.recordError(err, "initialize tensor {s} from {s}, shape={f}", .{ destination.name(), destination.parts[0].source.file_uri, destination.shape }); - return err; - }; - _ = loader.metrics.tensor_init_ns.fetchAdd(awakeNs(io) -| init_started, .monotonic); - } - const blocks = loader.credits.slotBlocks(self.slot)[0..job.blockLen(block_size)]; - switch (job.kind) { - .dma_read => |read| try self.readBlocks(scratch, blocks, read), - .dma_land => |land| try landJob(loader, self.plan, land, blocks, scratch), - // Reads nothing: its transfers read the zero block. - .dma_fill => return Dma.enqueue(self, loader.blocks[0..1]), - // A host job's worker writes its destinations itself, so the job - // ends with the worker's scheduling reference; its slot's blocks, - // if it reads through them, are done when this returns. - .host_direct => |read| { - var scatter: Scatter = .{ - .loader = loader, - .slot = self.plan.source_slot, - .iovecs = scratch.scatterIovecs(), - .file_offset = job.file_offset, - }; - for (transfers) |transfer| { - const tensor = transfer.destination.state.readyValue().?; - // The planner checked that the transfers cover the read - // in order, each source byte once. - try scatter.add(tensor.hostLanding(transfer)); - } - try scatter.flush(); - std.debug.assert(scatter.file_offset == job.file_offset + read.len); - }, - .host_bounce => |read| { - try self.readBlocks(scratch, blocks, read); - for (transfers) |transfer| { - const tensor = transfer.destination.state.readyValue().?; - // The last block holds stale bytes past the read, which - // no bounds check would stop a copy from taking. - std.debug.assert(transfer.landing + transfer.len <= read.len); - const piece = blockPiece(blocks, block_size, transfer.landing, transfer.len); - std.debug.assert(piece.len == transfer.len); - @memcpy(tensor.hostLanding(transfer), piece); - } - }, - .host_fill => { - for (transfers) |transfer| { - const tensor = transfer.destination.state.readyValue().?; - @memset(tensor.hostLanding(transfer), 0); - } - return; - }, - } - for (transfers) |transfer| transfer.destination.recordRead(transfer.len); - if (job.kind == .dma_read or job.kind == .dma_land) try Dma.enqueue(self, blocks); - } - - /// Reads the job's `read` into `blocks`, its slot's, in file order. - fn readBlocks(self: *ReadRequest, scratch: *Scratch, blocks: []const host_memory.Block, read: Plan.Job.Read) !void { - const loader = self.loader; - const block_size = loader.plan_config.block_size; - const iovecs = scratch.iovecs[0..blocks.len]; - for (iovecs, blocks, 0..) |*iovec, block, block_index| { - iovec.* = block[0..@min(block_size, read.len - block_index * block_size)]; - } - try readSlot(loader, self.plan.source_slot, iovecs, self.plan.jobs[self.index].file_offset, read.minimum_len); - } - - /// Releases a DMA piece or the scheduling reference. The final release - /// gives the credit back and may let the awaiting task free this request - /// and everything its batch owns. - fn release(self: *ReadRequest) void { - const previous = self.pending.fetchSub(1, .acq_rel); - std.debug.assert(previous > 0); - if (previous != 1) return; - - // Locals first: releasing the batch unit may complete the batch - // that owns this request, so nothing is touched after it. - const loader = self.loader; - const batch = self.plan.batch; - if (self.enqueued_ns != 0) { - _ = loader.metrics.dma_stage_ns.fetchAdd(awakeNs(loader.io) -| self.enqueued_ns, .monotonic); - } - loader.credits.release(loader.io, self.slot); - batch.releaseUnits(1); - } -}; - -/// Reads `[file_offset, file_offset + minimum)`, and possibly more, into -/// `buffers` in order, consuming them as it reads; bytes past `minimum` may -/// stay unread. The io takes as many buffers per call as it can (VFS's -/// direct read `IOV_MAX`, std's and zio's 8, a remote read all); a short -/// read resumes where it stopped. Reading stops once `minimum` bytes are -/// in, so the file may end only past `minimum`: the bytes after it are -/// padding a caller added to align the read, and the read that would fetch -/// them can start unaligned, which a file read directly refuses. Returns -/// the bytes read. -fn readFilePositionalAllV( - io: std.Io, - file: std.Io.File, - buffers: [][]u8, - file_offset: u64, - minimum: u64, -) !u64 { - var index: usize = 0; - var completed: u64 = 0; - while (completed < minimum) { - std.debug.assert(index < buffers.len); - const bytes_read = try file.readPositional(io, buffers[index..], file_offset + completed); - if (bytes_read == 0) return error.UnexpectedEndOfFile; - completed += bytes_read; - var rest = bytes_read; - while (index < buffers.len and rest >= buffers[index].len) : (index += 1) rest -= buffers[index].len; - if (rest != 0) buffers[index] = buffers[index][rest..]; - } - return completed; -} - -/// Every worker read goes through here, under a read permit: a failure is -/// recorded with the file, since a landing job's extents can come from files -/// other than its plan's. Once the loader failed, returns its first error. -/// Consumes `iovecs` (`readFilePositionalAllV`). -fn readSlot(loader: *Loader, slot: *SourceSlot, iovecs: [][]u8, file_offset: u64, minimum_len: usize) !void { - const io = loader.io; - const file = slot.ensure(io) catch |err| { - loader.recordError(err, "open {s}", .{slot.uri}); - return err; - }; - loader.read_permits.waitUncancelable(io); - defer loader.read_permits.post(io); - if (loader.first_error.get()) |err| return err; - const read_started = awakeNs(io); - const read_result = readFilePositionalAllV(io, file, iovecs, file_offset, minimum_len); - _ = loader.metrics.read_ns.fetchAdd(awakeNs(io) -| read_started, .monotonic); - const bytes_read = read_result catch |err| { - loader.recordError(err, "read {s} at {d}, {d} bytes at least", .{ slot.uri, file_offset, minimum_len }); - return err; - }; - _ = loader.metrics.read_operations.fetchAdd(1, .monotonic); - _ = loader.metrics.read_bytes.fetchAdd(@intCast(bytes_read), .monotonic); -} - -/// Reads a landing job's extents and lands their items' runs in its blocks -/// (`Plan.Item`, `walkItem`). An extent marked for it scatters -/// straight into the blocks, `max_positional_iovecs` ranges per call; any -/// other lands in the worker's bounce buffer and is copied from there: a -/// direct read, whose ranges must be aligned, and a gather read whose -/// source runs are out of order or repeat (a transpose, a broadcast). -fn landJob( - loader: *Loader, - plan: *const Plan, - land: Plan.Job.Land, - blocks: []const host_memory.Block, - scratch: *ReadRequest.Scratch, -) !void { - if (land.zeroed) { - for (blocks) |block| @memset(block[0..loader.plan_config.block_size], 0); - } - for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| { - try landExtent(loader, extent, plan.items[extent.item_start..][0..extent.item_len], plan.landings, blocks, scratch); - } -} - -fn landExtent( - loader: *Loader, - extent: Plan.Extent, - items: []const Plan.Item, - landings: []const u64, - blocks: []const host_memory.Block, - scratch: *ReadRequest.Scratch, -) !void { - const block_size = loader.plan_config.block_size; - if (extent.scatter) { - std.debug.assert(extent.len == extent.minimum_len); - var scatter: Scatter = .{ - .loader = loader, - .slot = extent.source_slot, - .iovecs = scratch.scatterIovecs(), - .file_offset = extent.file_offset, - .blocks = blocks, - }; - for (items) |item| try walkItem(item, landings, &scatter); - try scatter.flush(); - std.debug.assert(scatter.file_offset == extent.file_offset + extent.len); - return; - } - const bounce = try scratch.bounceBuffer(extent.len); - var buffers = [_][]u8{bounce}; - try readSlot(loader, extent.source_slot, &buffers, extent.file_offset, extent.minimum_len); - var sink: CopySink = .{ .bounce = bounce, .file_offset = extent.file_offset, .blocks = blocks, .block_size = block_size }; - for (items) |item| try copyItem(item, landings, &sink); -} - -/// Hands `sink` every run of `item` as `place(file offset, landing, len)`, -/// in run order, cut where the destination's box changes -/// (`ShardGrid.runEnd`). -fn walkItem(item: Plan.Item, landings: []const u64, sink: anytype) !void { - const table = landings[item.landing_start..][0..item.destination.grid.box_count]; - try walkRuns(item.lattice, item.first, item.end, item.clip_start, item.clip_end, &item.destination.grid, table, sink); -} - -/// `walkItem` for runs `first` to `end` of `lattice`, clipped to the -/// destination range `[clip_start, clip_end)`. -fn walkRuns(lattice: backend.Lattice, first: u64, end: u64, clip_start: u64, clip_end: u64, grid: *const ShardGrid, table: []const u64, sink: anytype) !void { - if (first >= end) return; - var runs: backend.Lattice.Cursor = .init(lattice, first, end); - while (runs.next()) |run| { - var at = @max(run.dst, clip_start); - const stop = @min(run.dst + lattice.width, clip_end); - var source = run.src + (at - run.dst); - while (at < stop) { - const piece_end = grid.runEnd(at, stop); - const box, const box_offset = grid.locate(at); - try sink.place(source, table[box] +% box_offset, piece_end - at); - source += piece_end - at; - at = piece_end; - } - } -} - -/// `walkItem` from the bounce buffer, a stretch at a time where that pays: -/// a stretch of the destination landing in one piece, inside one run of -/// the grid and one block, has its whole runs copied by -/// `Lattice.copyRuns`, and only the runs its ends cut go one at a time. A -/// stretch costs a few bisections, so items with few runs per stretch (a -/// part of a few wide runs, a grid that cuts every row) are walked. -fn copyItem(item: Plan.Item, landings: []const u64, sink: *CopySink) !void { - const lattice = item.lattice; - const grid = &item.destination.grid; - if (lattice.dims.len == 0 or item.first >= item.end) return walkItem(item, landings, sink); - const start = @max(item.clip_start, lattice.at(item.first).dst); - const stop = @min(item.clip_end, lattice.at(item.end - 1).dst + lattice.width); - if (start >= stop) return; - const stretches = grid.runCount(start, stop) + (stop - start) / sink.block_size + 1; - if (item.end - item.first < 16 * stretches) return walkItem(item, landings, sink); - const table = landings[item.landing_start..][0..grid.box_count]; - const source = @intFromPtr(sink.bounce.ptr) -% @as(usize, @intCast(sink.file_offset)); - var at = start; - while (at < stop) { - const box, const box_offset = grid.locate(at); - const landing: usize = @intCast(table[box] +% box_offset); - const in_block = landing % sink.block_size; - const stretch_end = @min(grid.runEnd(at, stop), at + (sink.block_size - in_block)); - // The address logical offset zero would have, were the whole - // destination laid out like this stretch. - const destination = @intFromPtr(sink.blocks[landing / sink.block_size].ptr) + in_block -% @as(usize, @intCast(at)); - const touched = @max(item.first, lattice.firstEndingAfter(at)); - const whole = @max(item.first, lattice.firstStartingFrom(at)); - const whole_end = @min(item.end, lattice.firstEndingAfter(stretch_end)); - const touched_end = @min(item.end, lattice.firstStartingFrom(stretch_end)); - if (whole < whole_end) { - try walkRuns(lattice, touched, whole, at, stretch_end, grid, table, sink); - lattice.copyRuns(whole, whole_end, destination, source); - try walkRuns(lattice, whole_end, touched_end, at, stretch_end, grid, table, sink); - } else { - try walkRuns(lattice, touched, touched_end, at, stretch_end, grid, table, sink); - } - at = stretch_end; - } -} - -/// The ranges a `Scatter` gathers before it reads, as many as one direct -/// read takes. -const max_positional_iovecs: usize = if (@TypeOf(std.posix.IOV_MAX) == void) 64 else std.posix.IOV_MAX; - -/// A read scattered over memory ranges as they come, in file order and -/// back to back from `file_offset` of `slot`'s file: a range that continues -/// the previous one in memory merges with it, and one call reads at most -/// `iovecs.len` ranges. Fed destination ranges directly (`add`), or the runs -/// of a walk landing in `blocks` (`place`). Every byte read must exist: a -/// scattered read has no alignment padding. -const Scatter = struct { - loader: *Loader, - slot: *SourceSlot, - iovecs: [][]u8, - count: usize = 0, - /// Bytes the pending ranges cover. - pending: usize = 0, - /// The file offset of the first byte not read yet. - file_offset: u64, - blocks: []const host_memory.Block = &.{}, - - fn add(self: *Scatter, range: []u8) !void { - if (self.count != 0) { - const last = &self.iovecs[self.count - 1]; - if (last.ptr + last.len == range.ptr) { - last.len += range.len; - self.pending += range.len; - return; - } - } - if (self.count == self.iovecs.len) try self.flush(); - self.iovecs[self.count] = range; - self.count += 1; - self.pending += range.len; - } - - /// A run's landing in the blocks, split at block boundaries. - fn place(self: *Scatter, source: u64, landing: u64, len: u64) !void { - std.debug.assert(source == self.file_offset + self.pending); - var at: usize = @intCast(landing); - var rest: usize = @intCast(len); - while (rest != 0) { - const piece = blockPiece(self.blocks, self.loader.plan_config.block_size, at, rest); - try self.add(piece); - at += piece.len; - rest -= piece.len; - } - } - - fn flush(self: *Scatter) !void { - if (self.count == 0) return; - try readSlot(self.loader, self.slot, self.iovecs[0..self.count], self.file_offset, self.pending); - self.file_offset += self.pending; - self.count = 0; - self.pending = 0; - } -}; - -/// Runs copied from a read in the bounce buffer, which starts at -/// `file_offset`. -const CopySink = struct { - bounce: []const u8, - file_offset: u64, - blocks: []const host_memory.Block, - block_size: usize, - - fn place(self: *CopySink, source: u64, landing: u64, len: u64) !void { - var at: usize = @intCast(landing); - var rest = self.bounce[@intCast(source - self.file_offset)..][0..@intCast(len)]; - while (rest.len != 0) { - const piece = blockPiece(self.blocks, self.block_size, at, rest.len); - @memcpy(piece, rest[0..piece.len]); - rest = rest[piece.len..]; - at += piece.len; - } - } -}; - -/// The bytes from offset `at` of `blocks` taken end to end, `block_size` -/// bytes each, up to `len` of them or the end of that block. -fn blockPiece(blocks: []const host_memory.Block, block_size: usize, at: usize, len: usize) []u8 { - const in_block = at % block_size; - return blocks[at / block_size][in_block..][0..@min(len, block_size - in_block)]; -} - -/// The DMA stage, over the loader's pumps (`Loader.pumps`): workers queue a -/// request's transfers per device, a pump submits them under the device's -/// in-flight budget, and their callbacks release the request. -const Dma = struct { - /// One DMA piece: `data`, in its request's slot, for `target` at - /// `destination_offset`. The piece holds one of its request's - /// references until its callback. Replicas share their source bytes: - /// copying them to another CPU socket first was rejected on MI300X, - /// where raw local and remote H2D both reached ~49-50 GiB/s per GPU - /// while one CPU thread copied across sockets at 5-10 GiB/s. - const Piece = struct { - target: *DestinationState.Target, - request: *ReadRequest, - data: []const u8, - destination_offset: usize, - }; - - /// Pieces ready for one device, submitted in arrival order so the - /// oldest requests and submissions complete first. Owned by the - /// device's `DevicePump.mutex`. - const ReadyQueue = std.Deque(Piece); - - /// One device's submission state. Its pump runs on whichever thread - /// finds work -- a worker after enqueueing, a ready callback after a - /// completion -- one at a time per device (`pumping`), and shares - /// nothing with the other devices' pumps. Per-device pumps load - /// replicated DeepSeek on four GB300 in 4.68 s, against 5.41 s with one - /// pump for every device, at unchanged pinned high-water. A dedicated - /// task per device took 5.0-5.4 s with a wake per completion (a futex - /// round trip per piece), or 4.4-5.1 s with half-budget wake hysteresis. - const DevicePump = struct { - mutex: std.Io.Mutex = .init, - queue: ReadyQueue = .empty, - /// In-flight DMA bytes and submissions; owned by `mutex`. - active_bytes: usize = 0, - active_pieces: usize = 0, - /// True while one thread runs this device's pump; owned by `mutex`. - /// One submitter per device is a correctness requirement: - /// `Target.nextIsLast` flags a tensor's last piece in submission - /// order. - pumping: bool = false, - /// Contexts whose callback fired, for the next pump or a batch's - /// retirement to destroy, never the event's own callback: an - /// intrusive stack through `EventContext.next_retired`, owned by - /// `mutex`. Destroying an event from the next pump instead of at its - /// batch's retirement bounds live PJRT events by the DMA width plus - /// one pump batch rather than by a submission's transfer count; - /// every shipped plugin accepted destruction outside the event's own - /// callback (16,384 fired events on two B70, CUDA and ROCm alike). - retired: ?*EventContext = null, - - /// Whether the next queued piece may go. The budget is in bytes, - /// not submissions: `dma_calibration.depth` blocks, and a piece is - /// one tensor's slice of a block, so a model of small tensors needs - /// many more pieces in flight to keep the same bytes moving: eight - /// average DeepSeek pieces occupy only ~18 MiB against the 128 MiB - /// budget of 16 MiB blocks. The piece that crosses the budget is - /// admitted: a device with room always has a piece in flight. - fn hasRoom(self: *const DevicePump, budget_bytes: usize) bool { - return self.active_bytes < budget_bytes and self.active_pieces < max_dma_pieces_per_device; - } - - /// `mutex`. Destroys every retired event. - fn destroyRetired(self: *DevicePump) void { - while (self.retired) |ctx| { - self.retired = ctx.next_retired; - ctx.next_retired = null; - ctx.destroyEvent(); - } - } - }; - - const EventContext = struct { - request: *ReadRequest, - /// Null once destroyed: by a pump or a batch's retirement after its - /// callback ran, or at once by a failed `onReady`. - pjrt_event: ?*pjrt.Event, - device_index: usize, - /// Bytes this submission holds against the device's in-flight budget. - len: usize, - /// Awake-clock nanoseconds of the submission. - submitted_ns: u64 = 0, - /// Link of the device pump's `retired`; owned by that pump's mutex. - next_retired: ?*EventContext = null, - - /// Pump mutex, unless no callback was registered. Destroys the - /// event, once. The platform is reached through the request, and - /// only when there is something to destroy: a context whose event - /// is null never touches it. - fn destroyEvent(self: *EventContext) void { - if (self.pjrt_event) |event| event.deinit(self.request.loader.platform.pjrt_api); - self.pjrt_event = null; - } - }; - - fn lockAllPumps(loader: *Loader) void { - for (loader.pumps) |*device_pump| device_pump.mutex.lockUncancelable(loader.io); - } - - fn unlockAllPumps(loader: *Loader) void { - var index = loader.pumps.len; - while (index > 0) { - index -= 1; - loader.pumps[index].mutex.unlock(loader.io); - } - } - - /// Retires the contexts of a done batch from the awaiting task, after - /// their callbacks fired, exactly as `pjrt.Event.await` does: drains - /// every pump's `retired`, whichever batch its contexts belong to, and - /// checks that every request completed. Runs with every pump locked so - /// an `abortReady` still iterating queued entries or a pump draining its - /// `retired` cannot race the free that follows. - fn retireBatch(loader: *Loader, batch: *Batch) void { - std.debug.assert(batch.done.isSet()); - lockAllPumps(loader); - defer unlockAllPumps(loader); - for (loader.pumps) |*device_pump| device_pump.destroyRetired(); - for (batch.plans.items) |plan| { - for (plan.requests) |*request| std.debug.assert(request.pending.load(.acquire) == 0); - } - } - - /// Hands a context whose callback fired to its device's next pump. Under - /// that pump's mutex, so the batch's retirement sees it before the batch - /// is freed. - fn retireEvent(loader: *Loader, ctx: *EventContext) void { - const device_pump = &loader.pumps[ctx.device_index]; - device_pump.mutex.lockUncancelable(loader.io); - ctx.next_retired = device_pump.retired; - device_pump.retired = ctx; - device_pump.mutex.unlock(loader.io); - } - - /// Queues every piece of the request's job before pumping: queuing piece - /// by piece costs roughly 69k-79k mutex and pump trips per DeepSeek load, - /// which coalesced reads alone do not remove. A transfer reads `len` - /// bytes from its `landing` of `blocks` taken end to end, inside one - /// block, once for each of its writers: the devices the planner planned - /// for (both come from the sharding's canonical device order). - fn enqueue(request: *ReadRequest, blocks: []const host_memory.Block) !void { - const loader = request.loader; - const block_size = loader.plan_config.block_size; - const transfers = request.plan.transfersOf(request.plan.jobs[request.index]); - var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - var pieces: usize = 0; - for (transfers) |transfer| { - const tensor = transfer.destination.state.readyValue().?; - pieces += @popCount(transfer.writer_mask); - var mask = transfer.writer_mask; - while (mask != 0) { - const writer_index: usize = @intCast(@ctz(mask)); - mask &= mask - 1; - counts[tensor.targets[writer_index].device_index] += 1; - } - } - const enqueued_ns = awakeNs(loader.io); - // Every pump is held while the request's pieces land, and every - // queue is reserved before any changes, so the request is either - // fully queued or not queued at all. - lockAllPumps(loader); - { - errdefer unlockAllPumps(loader); - // A failure recorded before these locks may have drained the - // queues already (`abortReady`), and nothing would complete what - // this queued; one recorded after them drains it. - if (loader.first_error.get()) |err| return err; - for (loader.pumps, counts[0..loader.pumps.len]) |*device_pump, count| { - try device_pump.queue.ensureUnusedCapacity(loader.allocator, count); - } - } - // Every piece holds the request, and with it the slot, until its - // callback. - _ = request.pending.fetchAdd(pieces, .acq_rel); - request.enqueued_ns = enqueued_ns; - for (transfers) |transfer| { - // The planner cut the transfers to the job's blocks. - const data = blockPiece(blocks, block_size, transfer.landing, transfer.len); - std.debug.assert(data.len == transfer.len); - const tensor = transfer.destination.state.readyValue().?; - var mask = transfer.writer_mask; - while (mask != 0) { - const writer_index: usize = @intCast(@ctz(mask)); - mask &= mask - 1; - const target = &tensor.targets[writer_index]; - loader.pumps[target.device_index].queue.pushBackAssumeCapacity(.{ - .target = target, - .request = request, - .data = data, - .destination_offset = transfer.destination_offset, - }); - } - } - unlockAllPumps(loader); - for (counts[0..loader.pumps.len], 0..) |count, device_index| { - if (count != 0) requestPump(loader, device_index); - } - } - - /// Runs the device's pump on this thread unless one is already running. - fn requestPump(loader: *Loader, device_index: usize) void { - const device_pump = &loader.pumps[device_index]; - device_pump.mutex.lockUncancelable(loader.io); - if (device_pump.pumping or loader.first_error.get() != null) { - device_pump.mutex.unlock(loader.io); - return; - } - device_pump.pumping = true; - device_pump.mutex.unlock(loader.io); - pump(loader, device_index); - } - - /// Submits from the device's queue, in arrival order, until the device - /// has no room or nothing is queued. Any queued piece can go: nothing - /// waits for another piece. Exactly one pump runs per device at a time, - /// see `DevicePump.pumping`. - fn pump(loader: *Loader, device_index: usize) void { - const device_pump = &loader.pumps[device_index]; - while (true) { - var selected: ?Piece = null; - device_pump.mutex.lockUncancelable(loader.io); - device_pump.destroyRetired(); - if (loader.first_error.get() == null) { - if (!device_pump.hasRoom(loader.dma_budget_bytes)) { - _ = loader.metrics.pump_stops_full.fetchAdd(1, .monotonic); - } else if (device_pump.queue.popFront()) |piece| { - device_pump.active_bytes += piece.data.len; - device_pump.active_pieces += 1; - selected = piece; - } else { - _ = loader.metrics.pump_stops_empty.fetchAdd(1, .monotonic); - } - } - if (selected == null) device_pump.pumping = false; - device_pump.mutex.unlock(loader.io); - if (selected) |piece| submitOne(loader, piece) else return; - } - } - - fn submitOne(loader: *Loader, piece: Piece) void { - const device_index = piece.target.device_index; - const len = piece.data.len; - submitTransfer(loader, piece) catch |err| { - loader.recordError(err, "submit DMA transfer: device={d}, destination_offset={d}, bytes={d}", .{ device_index, piece.destination_offset, len }); - finishPiece(loader, device_index, len, null, piece.request); - }; - } - - fn submitTransfer(loader: *Loader, piece: Piece) !void { - const api = loader.platform.pjrt_api; - const target = piece.target; - const is_last = target.nextIsLast(piece.data.len); - const submit_started = awakeNs(loader.io); - const event = try target.manager.transferData( - api, - 0, - piece.data, - @intCast(piece.destination_offset), - is_last, - ); - target.noteSubmitted(piece.data.len); - - // The plan holds one event slot per planned submission; the batch - // owns it. - const plan = piece.request.plan; - const event_index = plan.events_used.fetchAdd(1, .monotonic); - std.debug.assert(event_index < plan.events.len); - const ctx = &plan.events[event_index]; - ctx.* = .{ - .request = piece.request, - .pjrt_event = event, - .device_index = target.device_index, - .len = piece.data.len, - .submitted_ns = submit_started, - }; - - _ = loader.metrics.dma_submissions.fetchAdd(1, .monotonic); - event.onReady(api, EventContext, struct { - fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { - const request = ctx_.request; - const loader_ = request.loader; - const api_ = loader_.platform.pjrt_api; - const device_index = ctx_.device_index; - const len = ctx_.len; - _ = loader_.metrics.dma_piece_ns.fetchAdd(awakeNs(loader_.io) -| ctx_.submitted_ns, .monotonic); - // The shipped plugins resolve this event without an error - // whatever happened to the copy (the C API wrapper sets the - // promise with an OK status); kept for a plugin that reports. - // The callback owns the error. - if (err) |pjrt_error| { - loader_.recordError(pjrt_error.getCode(api_).toApiError(), "complete DMA transfer: device={d}, bytes={d}, message={s}", .{ device_index, len, pjrt_error.getMessage(api_) }); - pjrt_error.deinit(api_); - } - finishPiece(loader_, device_index, len, ctx_, request); - } - }.call, ctx) catch |err| { - event.awaitRaw(api) catch {}; - ctx.destroyEvent(); - return err; - }; - _ = loader.metrics.dma_submit_ns.fetchAdd(awakeNs(loader.io) -| submit_started, .monotonic); - } - - /// Ends a submitted piece: frees its room, which may pump on this thread; - /// then retires a fired context, after that pump, so its event is never - /// destroyed inside its own callback; then releases the request last, - /// since that release may complete and free the batch. - fn finishPiece(loader: *Loader, device_index: usize, len: usize, fired: ?*EventContext, request: *ReadRequest) void { - eventCompleted(loader, device_index, len); - if (fired) |ctx| retireEvent(loader, ctx); - request.release(); - } - - fn eventCompleted(loader: *Loader, device_index: usize, len: usize) void { - const device_pump = &loader.pumps[device_index]; - device_pump.mutex.lockUncancelable(loader.io); - std.debug.assert(device_pump.active_bytes >= len); - std.debug.assert(device_pump.active_pieces > 0); - device_pump.active_bytes -= len; - device_pump.active_pieces -= 1; - device_pump.mutex.unlock(loader.io); - // A ready callback can be the first place an asynchronous PJRT error - // becomes visible. Once outside the pump lock, retire every queued - // piece so request lifecycles cannot wait forever on entries that - // the failed pumps will no longer submit. - if (loader.first_error.get() != null) - abortReady(loader) - else - requestPump(loader, device_index); - } - - /// Force-completes every queued piece, one device at a time. A - /// completion here may finish its batch, but a batch still has an entry - /// queued until its last one is popped, and it is retired with every - /// pump locked, so no queued entry is touched after its request ends. - fn abortReady(loader: *Loader) void { - for (loader.pumps) |*device_pump| { - device_pump.mutex.lockUncancelable(loader.io); - while (device_pump.queue.popFront()) |piece| { - piece.request.release(); - } - device_pump.mutex.unlock(loader.io); - } - } -}; - -const DestinationState = struct { - /// One destination buffer of a tensor. PJRT makes the buffer ready once - /// every transfer submitted to it has completed and one of them carried - /// the last-transfer flag, whatever their completion order: the flag - /// only closes the buffer to further calls. The pump, the only - /// submitter, therefore flags the submission that completes the - /// placement's bytes, and no piece ever waits for another one. - /// Flagging the highest-offset piece instead makes a tail wait for its - /// prefixes, which can deadlock a sharded multi-device load: tails can - /// hold every lifecycle credit while their prefixes stay unclaimed, - /// whatever the pinned memory. This follows XLA's - /// CommonAsyncHostToDeviceTransferManager: readiness requires zero - /// in-flight transfers and a last-call flag, not destination-offset order. - const Target = struct { - manager: *pjrt.AsyncHostToDeviceTransferManager, - device_index: usize, - /// Bytes of the placement on this device. The pieces partition them: - /// those of every part, from whichever file and plan, and of the - /// zero fill. - total: usize, - /// Owned by the pump; read by the awaiting task once the batch is - /// done and nothing submits any more. - submitted_bytes: usize = 0, - - /// Whether a submission of `len` bytes closes the buffer. - fn nextIsLast(self: *const Target, len: usize) bool { - std.debug.assert(len != 0 and self.submitted_bytes + len <= self.total); - return self.submitted_bytes + len == self.total; - } - - fn noteSubmitted(self: *Target, len: usize) void { - std.debug.assert(self.submitted_bytes + len <= self.total); - self.submitted_bytes += len; - } - - /// Every byte went out, so the last-transfer flag did too. - fn fullySubmitted(self: *const Target) bool { - return self.submitted_bytes == self.total; - } - }; - - /// A DMA destination's transfer targets, one per device in canonical - /// order; empty for a host-resident one. - targets: []Target = &.{}, - /// A host-resident destination's shards in host memory, which workers - /// write in place, one per device in canonical order; empty for a DMA - /// one. The devices of one box share theirs (`initHost`). - host: [][*]u8 = &.{}, - - /// Creates the destination's buffers and writes the output shell: for a - /// DMA destination, the device buffers and transfer managers, one per - /// device of its grid in canonical order; for a host-resident one, the - /// buffers workers write (`initHost`). Runs once per destination - /// (`LazyOnce`), however many parts feed it, on the allocation task or - /// the worker that first reads or fills for it, which also counts the - /// allocation once. A host destination (`Memory.isHost`) is created in - /// each device's pinned host memory space and takes no device memory, - /// so it is not counted in `allocated_bytes`. - fn init(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { - if (destination.memory.isHost()) return initHost(direct, destination); - const allocator = direct.allocator; - const platform = direct.platform; - const grid = &destination.grid; - const targets = try allocator.alloc(Target, grid.devices.len); - errdefer allocator.free(targets); - - var pjrt_buffers: Buffer.Shards = .empty; - var initialized: usize = 0; - errdefer { - for (targets[0..initialized]) |target| target.manager.deinit(platform.pjrt_api); - for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(platform.pjrt_api); - } - - const shape_spec: pjrt.ShapeSpec = .init( - grid.local.dims(), - pjrtx.bufferTypeFromDtype(grid.local.dtype()), - ); - for (grid.devices, 0..) |device, i| { - const memory = platform.devices[device.id].memory(.default).?; - const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ - .shape_specs = &.{shape_spec}, - .memory = memory.pjrt_memory, - }); - errdefer manager.deinit(platform.pjrt_api); - const pjrt_buffer = try manager.retrieveBuffer(platform.pjrt_api, 0); - targets[i] = .{ - .manager = manager, - .device_index = device.id, - .total = grid.local.byteSize(), - }; - initialized += 1; - pjrt_buffers.appendAssumeCapacity(pjrt_buffer); - } - for (targets) |target| { - _ = direct.allocated_bytes[target.device_index].fetchAdd(target.total, .monotonic); - } - - destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); - return .{ .targets = targets }; - } - - /// Creates a host-resident destination's buffers, one per device of its - /// grid in canonical order, each in that device's pinned host memory - /// space, and reads their host pointers for the workers. The devices - /// holding one box (`ShardGrid.writers`) share one allocation: every - /// device's shard is a view of it, and the allocation lives until the - /// last view is gone (`SharedHostAllocation`). A transfer is then - /// written once, through its first writer's shard (`hostLanding`). A - /// `host_mapped` shard is always such a view, on the device's default - /// memory space, so kernels read the allocation in place. Every direct - /// target lets a host allocation of one device back a view on another, - /// for transfers and kernels alike: CPU, CUDA and ROCm allocate portable - /// pinned memory (`Buffer.HostAccessible.init`), and on oneAPI four B70 - /// gather from an allocation made in the first one's host memory space, - /// and read it back. A target that could not would need a copy per - /// device. - /// The buffers are complete before any worker writes them, so they - /// report ready from the start; the front end hands them out only once - /// the batch is done. - fn initHost(direct: *Loader, destination: *const Destination) StateInitError!DestinationState { - const allocator = direct.allocator; - const platform = direct.platform; - const api = platform.pjrt_api; - const grid = &destination.grid; - const shard_count = grid.devices.len; - std.debug.assert(shard_count <= Platform.MAX_NUM_DEVICES); - const shards = try allocator.alloc([*]u8, shard_count); - errdefer allocator.free(shards); - const dims = grid.local.dims(); - const element_type = pjrtx.bufferTypeFromDtype(grid.local.dtype()); - const layout = platform.defaultMemoryLayout(dims, grid.local.dtype()); - - // The writers of each device's box; the first of them creates the - // allocation the others view. - var box_writers: [Platform.MAX_NUM_DEVICES]u64 = undefined; - for (grid.writers[0..grid.box_count]) |writers| { - var mask = writers; - while (mask != 0) : (mask &= mask - 1) box_writers[@ctz(mask)] = writers; - } - - // Creation holds a reference to every shared allocation and its - // views one each. Declared before the buffers' cleanup, so a failure - // destroys the views first and this frees the owners. - var shared: [Platform.MAX_NUM_DEVICES]?*SharedHostAllocation = @splat(null); - defer for (shared[0..shard_count]) |allocation| if (allocation) |a| a.release(); - var pjrt_buffers: Buffer.Shards = .empty; - errdefer for (pjrt_buffers.constSlice()) |buffer| buffer.deinit(api); - for (grid.devices, shards, 0..) |device, *shard, index| { - // The front end only submits host kinds when every device has - // a pinned host space (`backend.supportsMemory`). - const host = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; - if (grid.replicas() > 1 or destination.memory == .host_mapped) { - const first: usize = @ctz(box_writers[index]); - if (first == index) shared[index] = try .create(direct.io, platform, dims, element_type, layout, host); - const allocation = shared[first].?; - const space = platform.devices[device.id].memory(destination.memory.space()).?.pjrt_memory; - pjrt_buffers.appendAssumeCapacity(try allocation.view(platform, dims, element_type, layout, space)); - shard.* = allocation.data; - } else { - const buffer = try createHostBuffer(direct.io, platform, dims, element_type, layout, host); - pjrt_buffers.appendAssumeCapacity(buffer); - shard.* = @ptrCast(try buffer.opaqueDeviceMemoryDataPointer(api)); - } - } - - destination.output.* = .fromPjrtBuffers(platform, destination.shape, pjrt_buffers.constSlice()); - return .{ .host = shards }; - } - - fn deinit(self: *DestinationState, allocator: std.mem.Allocator, api: *const pjrt.Api) void { - for (self.targets) |target| target.manager.deinit(api); - allocator.free(self.targets); - allocator.free(self.host); - } - - /// Where a host transfer's bytes go: the allocation its writers, the - /// devices of one box, share. Writing it once writes every one of them. - fn hostLanding(self: *const DestinationState, transfer: Plan.Transfer) []u8 { - const first: usize = @ctz(transfer.writer_mask); - std.debug.assert(first < self.host.len); - return self.host[first][transfer.destination_offset..][0..transfer.len]; - } -}; - -/// One host allocation that the devices of a box read through views of it, -/// in their pinned host memory space or, for `host_mapped`, their default -/// one: the owner buffer is not a shard, every view holds a reference that -/// its deletion callback drops, and the last drop destroys the owner. PJRT -/// runs a view's callback once the view's memory is out of use, after any -/// execution still reading it, so the allocation outlives its readers -/// whatever order the shards are freed in, including views of the shards -/// (`Buffer.createView`) that their owner outlives. On the process heap: -/// the last view may go long after the loader. -/// Writing through one view shows on every device, so a shard of a -/// replicated host-resident destination must not be donated to an -/// execution that writes it, as with `Buffer.HostAccessible.init(.undonatable)`. -const SharedHostAllocation = struct { - /// Allocations not released yet, counted in tests only. - var live: if (builtin.is_test) std.atomic.Value(usize) else void = if (builtin.is_test) .init(0) else {}; - - api: *const pjrt.Api, - owner: *pjrt.Buffer, - data: [*]u8, - /// One per live view, and one for `DestinationState.initHost` while it - /// creates them. - references: std.atomic.Value(usize) = .init(1), - - fn create( - io: std.Io, - platform: *const Platform, - dims: []const i64, - element_type: pjrt.BufferType, - layout: pjrt.MemoryLayout, - memory: *const pjrt.Memory, - ) StateInitError!*SharedHostAllocation { - const api = platform.pjrt_api; - const self = try std.heap.smp_allocator.create(SharedHostAllocation); - errdefer std.heap.smp_allocator.destroy(self); - const owner = try createHostBuffer(io, platform, dims, element_type, layout, memory); - errdefer owner.deinit(api); - self.* = .{ - .api = api, - .owner = owner, - .data = @ptrCast(try owner.opaqueDeviceMemoryDataPointer(api)), - }; - if (builtin.is_test) _ = live.fetchAdd(1, .monotonic); - return self; - } - - /// A shard in `memory`, holding a reference: the host space of its - /// device, or the default one, where kernels read the allocation in - /// place. Every device can read it. - fn view( - self: *SharedHostAllocation, - platform: *const Platform, - dims: []const i64, - element_type: pjrt.BufferType, - layout: pjrt.MemoryLayout, - memory: *const pjrt.Memory, - ) StateInitError!*pjrt.Buffer { - _ = self.references.fetchAdd(1, .monotonic); - errdefer self.release(); - return platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ - .device_buffer_ptr = self.data, - .dims = dims, - .element_type = element_type, - .layout = layout, - .memory = memory, - .on_delete_callback = &onViewDeleted, - .on_delete_callback_arg = self, - }); - } - - fn onViewDeleted(_: ?*anyopaque, context: ?*anyopaque) callconv(.c) void { - const self: *SharedHostAllocation = @ptrCast(@alignCast(context.?)); - self.release(); - } - - fn release(self: *SharedHostAllocation) void { - if (self.references.fetchSub(1, .acq_rel) != 1) return; - self.owner.deinit(self.api); - std.heap.smp_allocator.destroy(self); - if (builtin.is_test) _ = live.fetchSub(1, .monotonic); - } -}; - -/// Shared host allocations not released yet, in tests: the front end's -/// tests count the ones their views keep. -pub fn liveHostAllocations() usize { - comptime std.debug.assert(builtin.is_test); - return SharedHostAllocation.live.load(.acquire); -} - -/// An uninitialized buffer in a host memory space, once its allocation -/// completed: its host pointer is valid only then. -fn createHostBuffer( - io: std.Io, - platform: *const Platform, - dims: []const i64, - element_type: pjrt.BufferType, - layout: pjrt.MemoryLayout, - memory: *const pjrt.Memory, -) StateInitError!*pjrt.Buffer { - const api = platform.pjrt_api; - const buffer = try platform.pjrt_client.createUninitializedBuffer(api, .{ - .dims = dims, - .element_type = element_type, - .layout = layout, - .dst = .{ .memory = memory }, - }); - errdefer buffer.deinit(api); - const ready = buffer.readyEvent(api); - defer ready.deinit(api); - try ready.await(api, io); - return buffer; -} - -/// The lifecycle credits: one per request the pre-grown pinned set holds, -/// each a fixed slot of `blocks_per_slot` blocks. A worker takes a credit -/// before claiming a job, and the request gives it back with its last -/// reference: after its last DMA callback, or at the end of a host job. Only -/// the holder reads into, lands in and transfers from its slot's blocks, so -/// the credits are the one bound on pinned memory in use. A block whose -/// pieces are done stays with its request until the request ends; nobody -/// else could use it anyway, since every credit owns a whole slot. -const Credits = struct { - /// Every slot's blocks, slot `s` at `blocks[s * blocks_per_slot ..]`; - /// a tail shorter than a slot is left unused. - blocks: []const host_memory.Block, - blocks_per_slot: usize, - /// Free slots, a stack of indices; owned by `mutex`. - free: []u32, - free_len: usize, - /// The most slots held at once. - high_water: usize = 0, - mutex: std.Io.Mutex = .init, - /// Waiters for a slot; one release wakes one of them. - condition: std.Io.Condition = .init, - - fn init(allocator: std.mem.Allocator, blocks: []const host_memory.Block, blocks_per_slot: usize) std.mem.Allocator.Error!Credits { - std.debug.assert(blocks_per_slot != 0); - const free = try allocator.alloc(u32, blocks.len / blocks_per_slot); - // Slot 0 first. - for (free, 0..) |*slot, index| slot.* = @intCast(free.len - 1 - index); - return .{ .blocks = blocks, .blocks_per_slot = blocks_per_slot, .free = free, .free_len = free.len }; - } - - fn deinit(self: *Credits, allocator: std.mem.Allocator) void { - std.debug.assert(self.free_len == self.free.len); - allocator.free(self.free); - } - - fn count(self: *const Credits) usize { - return self.free.len; - } - - /// Waits for a free slot. Requests in flight give theirs back even after - /// a failure, so a waiter always gets one. - fn acquire(self: *Credits, io: std.Io) u32 { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - while (self.free_len == 0) { - self.condition.waitUncancelable(io, &self.mutex); - } - self.free_len -= 1; - self.high_water = @max(self.high_water, self.free.len - self.free_len); - return self.free[self.free_len]; - } - - fn release(self: *Credits, io: std.Io, slot: u32) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - std.debug.assert(self.free_len < self.free.len); - self.free[self.free_len] = slot; - self.free_len += 1; - // One release frees one slot: waking every waiter here would be a - // thundering herd. - self.condition.signal(io); - } - - fn slotBlocks(self: *const Credits, slot: u32) []const host_memory.Block { - return self.blocks[slot * self.blocks_per_slot ..][0..self.blocks_per_slot]; - } - - fn inUse(self: *Credits, io: std.Io) usize { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - return self.free.len - self.free_len; - } -}; - -/// The debug summary's counters (`Loader.logSummary`): read only by logs; -/// nothing here decides. -const Metrics = struct { - read_operations: std.atomic.Value(u64) = .init(0), - read_bytes: std.atomic.Value(u64) = .init(0), - dma_submissions: std.atomic.Value(u64) = .init(0), - /// Time workers spend waiting for a lifecycle credit and inside the - /// source read, summed over requests: waits above the read time mean the - /// load is DMA-completion bound. - credit_wait_ns: std.atomic.Value(u64) = .init(0), - read_ns: std.atomic.Value(u64) = .init(0), - /// From a request's enqueue to its last DMA callback: how long the DMA - /// stage holds a lifecycle credit and pinned blocks per request. - dma_stage_ns: std.atomic.Value(u64) = .init(0), - /// Inside `submitTransfer` (the PJRT call and the callback registration) - /// and from a submission to its ready callback, summed over submissions. - dma_submit_ns: std.atomic.Value(u64) = .init(0), - dma_piece_ns: std.atomic.Value(u64) = .init(0), - /// Why a device pump stopped: nothing queued, or no room under the - /// device's budget. A pump that mostly stops empty is starved by the - /// reads; one that mostly stops full is waiting on the engine. - pump_stops_empty: std.atomic.Value(u64) = .init(0), - pump_stops_full: std.atomic.Value(u64) = .init(0), - /// Inside `ensureState` for a claimed job's destinations: the first - /// worker to touch a tensor creates its PJRT buffers and transfer - /// managers there, and the other workers of the same tensor wait for it; - /// a host job waits there for the allocation task. - tensor_init_ns: std.atomic.Value(u64) = .init(0), - /// The allocation task's time creating host-resident destinations ahead - /// of their reads, summed over batches. - ahead_init_ns: std.atomic.Value(u64) = .init(0), -}; - -/// A value initialized at most once by whichever task touches it first, -/// with the initializer that task passes. Concurrent callers wait on the -/// event; a failed initialization keeps its error code and re-materializes -/// the same error for every later caller. -fn LazyOnce(comptime T: type) type { - return struct { - const Self = @This(); - const Status = enum(u8) { - uninitialized, - initializing, - ready, - failed, - }; - - value: T = undefined, - status: std.atomic.Value(Status) = .init(.uninitialized), - error_code: std.atomic.Value(u16) = .init(0), - initialized: std.Io.Event = .unset, - - /// The error set of `initFn`, which returns `T`. - fn InitError(comptime initFn: anytype) type { - return @typeInfo(@typeInfo(@TypeOf(initFn)).@"fn".return_type.?).error_union.error_set; - } - - fn ensure(self: *Self, io: std.Io, comptime initFn: anytype, args: std.meta.ArgsTuple(@TypeOf(initFn))) InitError(initFn)!*T { - while (true) switch (self.status.load(.acquire)) { - .uninitialized => { - if (self.status.cmpxchgStrong(.uninitialized, .initializing, .acq_rel, .acquire) != null) continue; - self.value = @call(.auto, initFn, args) catch |err| { - self.error_code.store(@intFromError(err), .release); - self.status.store(.failed, .release); - self.initialized.set(io); - return err; - }; - self.status.store(.ready, .release); - self.initialized.set(io); - return &self.value; - }, - .initializing => self.initialized.waitUncancelable(io), - .ready => return &self.value, - .failed => return @errorCast(@errorFromInt(self.error_code.load(.acquire))), - }; - } - - /// The value when initialization has completed successfully. - fn readyValue(self: *Self) ?*T { - return if (self.status.load(.acquire) == .ready) &self.value else null; - } - }; -} - -fn secondsBetween(from: std.Io.Timestamp, to: std.Io.Timestamp) f64 { - return @as(f64, @floatFromInt(from.durationTo(to).nanoseconds)) / std.time.ns_per_s; -} - -/// Mean milliseconds per unit of a summed duration (zero units: the total). -fn millisecondsPer(total_ns: u64, units: u64) f64 { - return @as(f64, @floatFromInt(total_ns)) / std.time.ns_per_ms / @as(f64, @floatFromInt(@max(units, 1))); -} - -fn awakeNs(io: std.Io) u64 { - const now: std.Io.Timestamp = .now(io, .awake); - return @intCast(@max(now.nanoseconds, 1)); -} - -/// Writes `contents` to a new file `name` in `dir` and returns its real -/// path, in `path_buffer`. -fn writeTestFile(dir: std.Io.Dir, io: std.Io, name: []const u8, contents: []const u8, path_buffer: []u8) ![]const u8 { - const file = try dir.createFile(io, name, .{ .read = true }); - defer file.close(io); - try file.writePositionalAll(io, contents, 0); - return path_buffer[0..try file.realPath(io, path_buffer)]; -} - -// Planner tests. - -/// Every file read at `alignment`, none opened to ask. -fn fixedReads(alignment: usize) Planner.Reads { - return .{ .io = undefined, .vfs = null, .policy = .off, .alignment = alignment }; -} - -/// What a loader's `Planner.Reads` predicts for `checkpoint` under `policy`. -fn testPrediction(io: std.Io, policy: VFS.DirectIo, checkpoint: ?[]const u8) !Planner.Reads.Prediction { - var reads: Planner.Reads = .init(io, policy); - defer reads.deinit(std.testing.allocator); - return reads.predict(std.testing.allocator, checkpoint); -} - -/// `Planner.preparePlan` with every file read at `alignment`, kept in -/// `arena`; the testing allocator checks the cut for leaks. -fn prepareTestPlan(arena: std.mem.Allocator, parts: []Part, order: []const usize, config: Planner.Config, alignment: usize) !*Plan { - return Planner.preparePlan(std.testing.allocator, arena, parts, order, config, fixedReads(alignment)); -} - -/// One plain destination per source, as `createDestinations` builds them -/// for whole-tensor specs, without device state: for planner tests. -fn initPlainParts( - platform: *const Platform, - sources: []safetensors.Tensor, - slots: []const *SourceSlot, - outputs: []Buffer, - destinations: []Destination, - parts: []Part, -) void { - for (sources, slots, outputs, destinations, parts) |*source, slot, *output, *destination, *part| { - part.* = .{ - .source = source, - .source_slot = slot, - .destination = destination, - .lattice = .{ .width = source.byteSize() }, - }; - destination.* = .{ - .shape = source.shape, - .grid = .init(platform, source.shape), - .output = output, - .parts = @as(*[1]Part, part), - .unread_bytes = .init(source.byteSize()), - }; - } -} - -/// Claims out and frees an open batch that a test only inspected; it must -/// be the only queued batch. -fn discardTestBatch(scheduler: *Scheduler, batch: *Batch) void { - const io = std.testing.io; - var claimed: usize = 0; - while (scheduler.claim(io)) |claim| { - std.debug.assert(claim.plan.batch == batch); - claimed += 1; - } - batch.releaseUnits(1 + claimed); - std.debug.assert(batch.done.isSet()); - batch.destroy(); -} - -test "the planner opens only local files to ask about direct I/O" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const contents: [2 * 4096]u8 = @splat(7); - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, io, "drafter.bin", &contents, &path_buffer); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - // A scheme whose files the plain io cannot open: asking would fail. - try vfs.registerBackend("remote", .{ .io = io }); - const vfs_io = vfs.io(); - const reads: Planner.Reads = .{ .io = vfs_io, .vfs = &vfs, .policy = .on, .alignment = 4096 }; - try std.testing.expectEqual(@as(usize, 0), reads.boundOf("remote://bucket/model.safetensors")); - try std.testing.expectEqual(@as(usize, 4096), reads.boundOf(path)); - - // A remote file is read exact, without an open on the planner. - var remote: SourceSlot = .{ .uri = "remote://bucket/model.safetensors" }; - defer remote.deinit(vfs_io); - try std.testing.expectEqual(@as(usize, 0), try reads.alignmentOf(&remote)); - try std.testing.expect(remote.file.readyValue() == null); - - // A local file beside it is opened and asked, whatever the checkpoint: - // direct where the filesystem allows it. - var local: SourceSlot = .{ .uri = path }; - defer local.deinit(vfs_io); - const alignment = try reads.alignmentOf(&local); - const file = local.file.readyValue().?.*; - try std.testing.expectEqual(@as(usize, if (vfs.useDirectIo(file, .on)) 4096 else 0), alignment); -} - -test "the checkpoint's reads are predicted as the VFS will decide the files its registry reads" { - if (VFS.direct_io_alignment == 0) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const long: [2 * VFS.direct_io_alignment + 100]u8 = @splat(7); - try tmp.dir.writeFile(io, .{ .sub_path = "a.safetensors", .data = &long }); - try tmp.dir.writeFile(io, .{ .sub_path = "b.safetensors", .data = &long }); - try tmp.dir.writeFile(io, .{ .sub_path = "c.safetensors", .data = long[0..100] }); - // The index lists a, b, c and a symlink to a; the registry does not - // read the consolidated file beside them (Mistral's format). The files - // hold no header: none is read. - try tmp.dir.symLink(io, "a.safetensors", "alias.safetensors", .{}); - try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"x\":\"a.safetensors\",\"y\":\"b.safetensors\",\"z\":\"c.safetensors\",\"w\":\"alias.safetensors\"}}" }); - try tmp.dir.writeFile(io, .{ .sub_path = "consolidated.safetensors", .data = &long }); - // A snapshot whose file links into blobs, as a Hugging Face cache's do. - const blob: [8 * 1024]u8 = @splat(9); - try tmp.dir.createDir(io, "blobs", .default_dir); - try tmp.dir.writeFile(io, .{ .sub_path = "blobs/x", .data = &blob }); - try tmp.dir.createDir(io, "snapshot", .default_dir); - try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); - var root_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; - var path_buffer: [std.fs.max_path_bytes + 64]u8 = undefined; - - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - // A remote scheme: nothing is predicted for a checkpoint there. - try vfs.registerBackend("remote", .{ .io = io }); - const vfs_io = vfs.io(); - - // The listed files as fresh handles decide them under `on`: c, shorter - // than a unit, never direct. - var expected: Planner.Reads.Prediction = .{}; - const dir = try std.Io.Dir.openDir(.cwd(), vfs_io, root, .{}); - defer dir.close(vfs_io); - for ([_][]const u8{ "a.safetensors", "b.safetensors", "c.safetensors" }) |name| { - const file = try dir.openFile(vfs_io, name, .{}); - defer file.close(vfs_io); - const size = (try file.stat(vfs_io)).size; - expected.files += 1; - expected.bytes += size; - if (vfs.useDirectIo(file, .on)) expected.direct_bytes += size; - } - try std.testing.expectEqual(@as(u64, 2 * long.len + 100), expected.bytes); - - var on_reads: Planner.Reads = .init(vfs_io, .on); - defer on_reads.deinit(allocator); - // The symlink is predicted once, with a. - try std.testing.expectEqual(expected, try on_reads.predict(allocator, root)); - // Each answer is kept by the file's real path, none for the - // consolidated file. Under `on` each keeps `on`, c too: planning - // decides it, and it reads buffered. - try std.testing.expectEqual(@as(usize, 3), on_reads.predicted.count()); - try std.testing.expectEqual(VFS.DirectIo.on, on_reads.predicted.get(try std.fmt.bufPrint(&path_buffer, "{s}/c.safetensors", .{root})).?); - try std.testing.expectEqual(expected, try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors.index.json", .{root}))); - const one = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/a.safetensors", .{root})); - try std.testing.expectEqual(@as(usize, 1), one.files); - try std.testing.expectEqual(@as(u64, long.len), one.bytes); - const snapshot = try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root})); - try std.testing.expectEqual(@as(usize, 1), snapshot.files); - try std.testing.expectEqual(@as(u64, blob.len), snapshot.bytes); - - // Just written, so cached: none reads direct under `auto`, and each is - // kept as buffered. - var auto_reads: Planner.Reads = .init(vfs_io, .auto); - defer auto_reads.deinit(allocator); - const cached = try auto_reads.predict(allocator, root); - try std.testing.expectEqual(@as(usize, 3), cached.files); - try std.testing.expectEqual(@as(u64, 0), cached.direct_bytes); - var answers = auto_reads.predicted.valueIterator(); - while (answers.next()) |answer| try std.testing.expectEqual(VFS.DirectIo.off, answer.*); - - // Nothing to predict: no direct I/O, no VFS, a remote or no checkpoint, - // or none there. - const nothing = [_]Planner.Reads.Prediction{ - try testPrediction(vfs_io, .off, root), - try testPrediction(io, .on, root), - try testPrediction(vfs_io, .on, "remote://bucket/model"), - try testPrediction(vfs_io, .on, null), - try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/missing", .{root})), - try testPrediction(vfs_io, .on, try std.fmt.bufPrint(&path_buffer, "{s}/blobs", .{root})), - }; - for (nothing) |prediction| try std.testing.expectEqual(@as(usize, 0), prediction.files); -} - -test "a file predicted to read direct is planned direct, though the prediction's sample cached it" { - if (comptime VFS.direct_io_alignment == 0) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - // A snapshot whose file links into blobs, as a Hugging Face cache's - // does. 64 KiB: every page of it is sampled, on 4 KiB pages too. - const data: [64 * 1024]u8 = @splat(7); - const blobs = try tmp.dir.createDirPathOpen(io, "blobs", .{}); - defer blobs.close(io); - try testing.writeSafetensorsFile(allocator, io, blobs, "x", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{data.len}, .bytes = &data }, - }); - try tmp.dir.createDir(io, "snapshot", .default_dir); - try tmp.dir.symLink(io, "../blobs/x", "snapshot/model.safetensors", .{}); - var root_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; - var path_buffer: [std.fs.max_path_bytes + 16]u8 = undefined; - const checkpoint = try std.fmt.bufPrint(&path_buffer, "{s}/snapshot", .{root}); - - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - const vfs_io = vfs.io(); - // The planner's slot takes the registry's name for the file: the blob. - var registry: safetensors.TensorRegistry = try .fromPath(allocator, vfs_io, checkpoint); - defer registry.deinit(); - const uri = registry.tensors.get("w").?.file_uri; - { - const file = try blobs.openFile(io, "x", .{}); - defer file.close(io); - try file.sync(io); - _ = std.os.linux.fadvise(file.handle, 0, 0, std.os.linux.POSIX_FADV.DONTNEED); - } - - var reads: Planner.Reads = .init(vfs_io, .auto); - defer reads.deinit(allocator); - const predicted = try reads.predict(allocator, checkpoint); - // A filesystem that kept the file cached (tmpfs) or refuses direct - // reads says nothing here. - if (predicted.direct_bytes == 0) return error.SkipZigTest; - // Once the sample's reads landed, a probe of its own finds the file - // cached (a kernel that queues no read for a missed sample shows - // nothing). - try io.sleep(.fromMilliseconds(250), .awake); - { - const file = try std.Io.Dir.openFile(.cwd(), vfs_io, uri, .{ .mode = .read_only }); - defer file.close(vfs_io); - if (vfs.useDirectIo(file, .auto)) return error.SkipZigTest; - } - - // Planning reads it as predicted: direct, the handle decided so. - var slot: SourceSlot = .{ .uri = uri }; - defer slot.deinit(vfs_io); - try std.testing.expectEqual(reads.alignment, try reads.alignmentOf(&slot)); - try std.testing.expect(vfs.useDirectIo(slot.file.readyValue().?.*, .auto)); -} - -test "arenas are placed for direct reads from four fifths of the checkpoint's bytes" { - const Case = struct { direct_bytes: u64, bytes: u64, direct: bool }; - const cases = [_]Case{ - .{ .direct_bytes = 0, .bytes = 0, .direct = false }, - .{ .direct_bytes = 4, .bytes = 5, .direct = true }, - .{ .direct_bytes = 79, .bytes = 100, .direct = false }, - .{ .direct_bytes = 80, .bytes = 100, .direct = true }, - .{ .direct_bytes = 0, .bytes = 100, .direct = false }, - }; - for (cases) |case| { - const prediction: Planner.Reads.Prediction = .{ .bytes = case.bytes, .direct_bytes = case.direct_bytes }; - try std.testing.expectEqual(case.direct, prediction.direct()); - } -} - -test "a strided part whose source runs are not back to back counts its whole source range" { - // [32, 16] u8 at a file offset off a 64-byte alignment. - const source: safetensors.Tensor = .{ .file_uri = "wide.bin", .name = "wide", .shape = .init(.{ 32, 16 }, .u8), .offset = 100 }; - const config: Planner.Config = .{ .block_size = 256, .request_size = 512 }; - // Transposed, 512 runs of one byte: every job reads all 512 bytes, - // 576 widened to the alignment. - const transposed: LoadPart = .{ .source = &source, .lattice = .{ .width = 1, .dims = &.{ - .{ .count = 16, .src_stride = 1, .dst_stride = 32 }, - .{ .count = 32, .src_stride = 16, .dst_stride = 1 }, - } } }; - try std.testing.expectEqual(@as(backend.Backend.StridedRead, .{ .read = 512, .limit = 512 }), Planner.stridedRead(config, fixedReads(0), transposed)); - try std.testing.expectEqual(576, Planner.stridedRead(config, fixedReads(64), transposed).read); - // A bigger block calibration picked does not raise the limit past the - // smallest one's request. - const bigger_block: Planner.Config = .{ .block_size = 1024, .request_size = 1024, .gather_limit = 512 }; - try std.testing.expectEqual(512, Planner.stridedRead(bigger_block, fixedReads(0), transposed).limit); - // Every other row, in order: the gaps stretch a job's range, so the - // whole range counts, from the first row to the last. - const every_other_row: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 16, .src_stride = 32, .dst_stride = 16 }} } }; - try std.testing.expectEqual(15 * 32 + 16, Planner.stridedRead(config, fixedReads(0), every_other_row).read); - // Rows back to back read within their job's range. - const rows: LoadPart = .{ .source = &source, .lattice = .{ .width = 16, .dims = &.{.{ .count = 32, .src_stride = 16, .dst_stride = 16 }} } }; - try std.testing.expectEqual(0, Planner.stridedRead(config, fixedReads(64), rows).read); -} - -test "source planner coalesces exact adjacent and overlapping tensor ranges per file" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - - var sources = [_]safetensors.Tensor{ - .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 10 }, - .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 14 }, - .{ .file_uri = "a", .name = "a0-copy", .shape = .init(.{4}, .u8), .offset = 10 }, - .{ .file_uri = "a", .name = "a-gap", .shape = .init(.{4}, .u8), .offset = 20 }, - .{ .file_uri = "b", .name = "b0", .shape = .init(.{12}, .u8), .offset = 3 }, - }; - var slots = [_]SourceSlot{ - .{ .uri = "a" }, - .{ .uri = "b" }, - }; - var outputs: [sources.len]Buffer = undefined; - var destinations: [sources.len]Destination = undefined; - var parts: [sources.len]Part = undefined; - initPlainParts(platform, &sources, &.{ &slots[0], &slots[0], &slots[0], &slots[0], &slots[1] }, &outputs, &destinations, &parts); - - var scheduler: Scheduler = .init(allocator); - defer scheduler.deinit(); - const batch = try Batch.create(allocator, io, 0); - try Planner.publishFiles(&scheduler, io, batch, &parts, .{ - .block_size = 4, - .request_size = 8, - }, fixedReads(0)); - - // One plan per file. a:[10,18) merges adjacency and the duplicate, - // a:[20,24) remains exact, and b:[3,15) is split at the request-size - // boundary. - const plans = batch.plans.items; - try std.testing.expectEqual(@as(usize, 2), plans.len); - try std.testing.expect(plans[0].source_slot == &slots[0]); - try std.testing.expect(plans[1].source_slot == &slots[1]); - try std.testing.expectEqual(@as(usize, 2), plans[0].jobs.len); - try std.testing.expectEqual(@as(usize, 4), plans[0].transfers.len); - try std.testing.expectEqual(@as(u64, 10), plans[0].jobs[0].file_offset); - try std.testing.expectEqual(@as(usize, 8), plans[0].jobs[0].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 3), plans[0].jobs[0].transfer_len); - try std.testing.expectEqual(@as(u64, 20), plans[0].jobs[1].file_offset); - try std.testing.expectEqual(@as(usize, 2), plans[1].jobs.len); - try std.testing.expectEqual(@as(usize, 3), plans[1].transfers.len); - try std.testing.expectEqual(@as(u64, 3), plans[1].jobs[0].file_offset); - try std.testing.expectEqual(@as(usize, 8), plans[1].jobs[0].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 4), plans[1].jobs[1].kind.dma_read.len); - try std.testing.expectEqual(@as(u64, 12), plans[0].source_bytes); - try std.testing.expectEqual(@as(u64, 12), plans[1].source_bytes); - // One writer per transfer on one device: one event per transfer. Jobs - // span their 4-byte blocks. - try std.testing.expectEqual(@as(usize, 4), plans[0].events.len); - try std.testing.expectEqual(@as(usize, 3), plans[1].events.len); - try std.testing.expectEqual(@as(usize, 2), plans[0].requests.len); - try std.testing.expectEqual(@as(usize, 2), plans[0].jobs[0].blockLen(4)); - try std.testing.expectEqual(@as(usize, 1), plans[0].jobs[1].blockLen(4)); - try std.testing.expectEqual(@as(usize, 2), plans[1].jobs[0].blockLen(4)); - try std.testing.expectEqual(@as(usize, 1), plans[1].jobs[1].blockLen(4)); - for (plans) |plan| { - for (plan.requests) |*request| try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); - } - discardTestBatch(&scheduler, batch); - - var aligned_sources = [_]safetensors.Tensor{ - .{ .file_uri = "aligned", .name = "aligned0", .shape = .init(.{7}, .u8), .offset = 0 }, - .{ .file_uri = "aligned", .name = "aligned1", .shape = .init(.{8}, .u8), .offset = 7 }, - .{ .file_uri = "aligned", .name = "aligned2", .shape = .init(.{5}, .u8), .offset = 15 }, - }; - var aligned_slot: SourceSlot = .{ .uri = "aligned" }; - var aligned_outputs: [aligned_sources.len]Buffer = undefined; - var aligned_destinations: [aligned_sources.len]Destination = undefined; - var aligned_items: [aligned_sources.len]Part = undefined; - initPlainParts(platform, &aligned_sources, &.{ &aligned_slot, &aligned_slot, &aligned_slot }, &aligned_outputs, &aligned_destinations, &aligned_items); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const aligned_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ - .block_size = 4, - .request_size = 8, - }, 0); - try std.testing.expectEqual(@as(usize, 3), aligned_plan.jobs.len); - try std.testing.expectEqual(@as(usize, 6), aligned_plan.transfers.len); - try std.testing.expectEqual(@as(usize, 7), aligned_plan.jobs[0].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 8), aligned_plan.jobs[1].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 5), aligned_plan.jobs[2].kind.dma_read.len); - - // Widened for direct I/O: the same three jobs (16 - 2 * 4 = 8 still cuts - // at the tensor-safe boundaries 7 and 15), each read at 4-byte bounds, - // its transfers relative to the widened start; the last read runs past - // the tensor data, of which every byte is required. - const widened_plan = try prepareTestPlan(arena.allocator(), &aligned_items, &.{ 0, 1, 2 }, .{ - .block_size = 4, - .request_size = 16, - }, 4); - try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs.len); - try std.testing.expectEqual(@as(u64, 0), widened_plan.jobs[0].file_offset); - try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[0].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 7), widened_plan.jobs[0].kind.dma_read.minimum_len); - try std.testing.expectEqual(@as(u64, 4), widened_plan.jobs[1].file_offset); - try std.testing.expectEqual(@as(usize, 12), widened_plan.jobs[1].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 11), widened_plan.jobs[1].kind.dma_read.minimum_len); - try std.testing.expectEqual(@as(usize, 3), widened_plan.jobs[1].blockLen(4)); - try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[1].transfer_start].landing); - try std.testing.expectEqual(@as(u64, 12), widened_plan.jobs[2].file_offset); - try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.len); - try std.testing.expectEqual(@as(usize, 8), widened_plan.jobs[2].kind.dma_read.minimum_len); - try std.testing.expectEqual(@as(usize, 3), widened_plan.transfers[widened_plan.jobs[2].transfer_start].landing); - try std.testing.expectEqual(@as(u64, 20), widened_plan.source_bytes); -} - -test "host-resident and DMA parts never share a job, and overlapping host parts read through blocks" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 2 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - // x, y and z end to end in one file. x lands in two host destinations, - // y in device memory between them, z in a third host destination. - const shape = Shape.init(.{ 8, 8 }, .u8); - var sources = [_]safetensors.Tensor{ - .{ .file_uri = "file", .name = "x", .shape = shape, .offset = 0 }, - .{ .file_uri = "file", .name = "y", .shape = shape, .offset = 64 }, - .{ .file_uri = "file", .name = "z", .shape = shape, .offset = 128 }, - }; - var slot: SourceSlot = .{ .uri = "file" }; - var outputs: [4]Buffer = undefined; - var destinations: [4]Destination = undefined; - var parts = [_]Part{ - .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[0], .lattice = .{ .width = 64 } }, - .{ .source = &sources[1], .source_slot = &slot, .destination = &destinations[1], .lattice = .{ .width = 64 } }, - .{ .source = &sources[0], .source_slot = &slot, .destination = &destinations[2], .lattice = .{ .width = 64 } }, - .{ .source = &sources[2], .source_slot = &slot, .destination = &destinations[3], .lattice = .{ .width = 64 } }, - }; - const memories = [_]Memory{ .host_pinned, .default, .host_pinned, .host_pinned }; - for (&destinations, &outputs, &parts, memories) |*destination, *output, *part, memory| { - destination.* = .{ .shape = shape, .grid = .init(platform, shape), .memory = memory, .output = output, .parts = @as(*[1]Part, part), .unread_bytes = .init(64) }; - } - - const order = try Planner.sortedPartOrder(allocator, &parts); - defer allocator.free(order); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const plan = try prepareTestPlan(arena.allocator(), &parts, order, .{ - .block_size = 64, - .request_size = 1024, - }, 0); - // x's run reads its bytes once for two destinations, so through - // blocks; y goes out by DMA; z is read in place. File order. - const expected = [_]struct { offset: u64, kind: std.meta.Tag(Plan.Job.Kind) }{ - .{ .offset = 0, .kind = .host_bounce }, - .{ .offset = 64, .kind = .dma_read }, - .{ .offset = 128, .kind = .host_direct }, - }; - try std.testing.expectEqual(expected.len, plan.jobs.len); - var dma_writers: usize = 0; - for (plan.jobs, expected) |job, want| { - try std.testing.expectEqual(want.offset, job.file_offset); - try std.testing.expectEqual(want.kind, std.meta.activeTag(job.kind)); - for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { - try std.testing.expectEqual(job.kind != .dma_read, transfer.destination.memory.isHost()); - if (job.kind == .dma_read) dma_writers += @popCount(transfer.writer_mask); - } - } - try std.testing.expectEqual(@as(usize, 2), plan.jobs[0].transfer_len); - try std.testing.expectEqual(@as(usize, 1), plan.jobs[0].blockLen(64)); - try std.testing.expectEqual(dma_writers, plan.events.len); -} - -/// Plans a composite destination fed by three parts from two files, with a -/// gap and a tail of zero fill, beside a plain one, then replays the plans on -/// host memory: every device's placement bytes are written exactly once, -/// from the right source bytes or from zeros. -const CompositeTest = struct { - const Options = struct { - block_size: usize, - request_size: usize, - memory: Memory = .default, - alignment: usize = 0, - /// By default nothing is staged. - staging_gain: usize = std.math.maxInt(usize), - /// Whether some piece holds a's last byte and b's first one. - cross_part_piece: bool, - }; - - /// With a host memory kind the destinations take host jobs: direct - /// without `alignment`, through its slot's blocks with it. A DMA job is - /// staged where it saves `staging_gain` submissions; the replay reads - /// its blocks from the two files as its worker would (`jobBlocks`). - fn run(platform: *const Platform, shape: Shape, options: Options) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - const block_size = options.block_size; - var first_bytes: [96]u8 = undefined; - for (&first_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 31 +% 7)) | 1; - var second_bytes: [128]u8 = undefined; - for (&second_bytes, 0..) |*byte, i| byte.* = @as(u8, @truncate(i *% 17 +% 3)) | 0x80; - // Rows of 16 bytes. a and b are adjacent in "first" and in the - // destination; c leaves a five-row gap before it and a two-row tail. - var sources = [_]safetensors.Tensor{ - .{ .file_uri = "first", .name = "a", .shape = .init(.{ 3, 8 }, .u16), .offset = 4 }, - .{ .file_uri = "first", .name = "b", .shape = .init(.{ 2, 8 }, .u16), .offset = 52 }, - .{ .file_uri = "second", .name = "c", .shape = .init(.{ 4, 8 }, .u16), .offset = 0 }, - .{ .file_uri = "second", .name = "plain", .shape = .init(.{ 4, 8 }, .u16), .offset = 64 }, - }; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var paths: [2][std.Io.Dir.max_path_bytes]u8 = undefined; - var slots: [2]SourceSlot = undefined; - var opened: usize = 0; - defer for (slots[0..opened]) |*slot| slot.deinit(io); - for (&slots, &paths, [_][]const u8{ "first", "second" }, [_][]const u8{ &first_bytes, &second_bytes }) |*slot, *path, name, bytes| { - slot.* = .{ .uri = try writeTestFile(tmp.dir, io, name, bytes, path) }; - opened += 1; - } - var fixture: TestLoader = undefined; - try fixture.init(1); - defer fixture.deinit(); - const loader = &fixture.loader; - loader.plan_config = .{ .block_size = block_size, .request_size = options.request_size }; - var outputs: [2]Buffer = undefined; - var destinations: [2]Destination = undefined; - var parts = [_]Part{ - .{ .source = &sources[0], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 0, .width = 48 } }, - .{ .source = &sources[1], .source_slot = &slots[0], .destination = &destinations[0], .lattice = .{ .dst_offset = 48, .width = 32 } }, - .{ .source = &sources[2], .source_slot = &slots[1], .destination = &destinations[0], .lattice = .{ .dst_offset = 160, .width = 64 } }, - .{ .source = &sources[3], .source_slot = &slots[1], .destination = &destinations[1], .lattice = .{ .width = 64 } }, - }; - const plain_shape = shape.setDim(0, 4); - destinations = .{ - .{ .shape = shape, .grid = .init(platform, shape), .memory = options.memory, .output = &outputs[0], .parts = parts[0..3], .unread_bytes = .init(144) }, - .{ .shape = plain_shape, .grid = .init(platform, plain_shape), .memory = options.memory, .output = &outputs[1], .parts = parts[3..4], .unread_bytes = .init(64) }, - }; - const host = destinations[0].memory.isHost(); - - const order = try Planner.sortedPartOrder(allocator, &parts); - defer allocator.free(order); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - var plans: [2]*Plan = undefined; - var plan_count: usize = 0; - var start: usize = 0; - while (start < order.len) : (plan_count += 1) { - const end = Planner.fileGroupEnd(&parts, order, start); - plans[plan_count] = try prepareTestPlan(arena.allocator(), &parts, order[start..end], .{ - .block_size = block_size, - .request_size = options.request_size, - .minimum_staging_gain = options.staging_gain, - }, options.alignment); - start = end; - } - try std.testing.expectEqual(@as(usize, 2), plan_count); - // The fill goes out once, with the plan of the destination's first - // part, as its last job. - const fill_kind: std.meta.Tag(Plan.Job.Kind) = if (host) .host_fill else .dma_fill; - try std.testing.expectEqual(fill_kind, std.meta.activeTag(plans[0].jobs[plans[0].jobs.len - 1].kind)); - // Host jobs take no DMA event; DMA jobs one event per writer. - for (plans[0..plan_count], [_]usize{ 1, 0 }) |plan, fill_jobs| { - var writers: usize = 0; - var fills: usize = 0; - for (plan.jobs) |job| { - const kind = std.meta.activeTag(job.kind); - if (kind == fill_kind) fills += 1; - if (host) { - const read_kind: std.meta.Tag(Plan.Job.Kind) = if (options.alignment == 0) .host_direct else .host_bounce; - try std.testing.expect(kind == .host_fill or kind == read_kind); - } else { - try std.testing.expect(kind == .dma_read or kind == .dma_land or kind == .dma_fill); - } - for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| writers += @popCount(transfer.writer_mask); - } - try std.testing.expectEqual(fill_jobs, fills); - try std.testing.expectEqual(if (host) 0 else writers, plan.events.len); - } - - const zeros = try allocator.alloc(u8, 256); - defer allocator.free(zeros); - @memset(zeros, 0); - var replays: [2]Replay = undefined; - var initialized: usize = 0; - defer for (replays[0..initialized]) |*replay| replay.deinit(allocator); - for (&replays, &destinations) |*replay, *destination| { - replay.* = try .init(allocator, platform.placedShape(destination.shape).packedShape()); - initialized += 1; - } - var cross_part_piece = false; - for (plans[0..plan_count]) |plan| { - const file_bytes: []const u8 = if (plan.source_slot == &slots[0]) &first_bytes else &second_bytes; - for (plan.jobs) |job| { - const fill = job.kind == .dma_fill or job.kind == .host_fill; - const blocks = if (host or fill) &.{} else try jobBlocks(allocator, loader, plan, job); - defer allocator.free(blocks); - for (plan.transfers[job.transfer_start..][0..job.transfer_len]) |transfer| { - // Every transfer stays inside one block (`Plan.Transfer`). - try std.testing.expect(transfer.landing % block_size + transfer.len <= block_size); - switch (job.kind) { - .host_direct, .host_bounce => |read| try std.testing.expect(transfer.landing + transfer.len <= read.len), - else => try std.testing.expect(transfer.landing < (if (fill) 1 else job.blockLen(block_size)) * block_size), - } - const bytes = if (fill) fill: { - try std.testing.expectEqual(@as(usize, 0), transfer.landing); - break :fill zeros[0..transfer.len]; - } else if (host) read: { - // A host job's landings address its read, which - // starts at the job's file offset. - const file_offset = job.file_offset + transfer.landing; - if (plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; - break :read file_bytes[file_offset..][0..transfer.len]; - } else read: { - // a's last byte and b's first one in the same piece, - // where a read's blocks hold the file in order. - const file_offset = job.file_offset + transfer.landing; - if (job.kind == .dma_read and plan.source_slot == &slots[0] and file_offset <= 51 and file_offset + transfer.len > 52) cross_part_piece = true; - break :read blocks[transfer.landing..][0..transfer.len]; - }; - const index: usize = if (transfer.destination == &destinations[0]) 0 else 1; - try replays[index].write(transfer.writer_mask, transfer.destination_offset, bytes); - } - } - } - try std.testing.expectEqual(options.cross_part_piece, cross_part_piece); - - var global: [256]u8 = @splat(0); - @memcpy(global[0..80], first_bytes[4..84]); - @memcpy(global[160..224], second_bytes[0..64]); - try replays[0].expect(allocator, &global); - try replays[1].expect(allocator, second_bytes[64..128]); - } - - /// A DMA job's blocks taken end to end once its worker has read it: - /// the read in file order, or as its worker lands a landing job - /// (`landJob`). - fn jobBlocks(allocator: std.mem.Allocator, loader: *Loader, plan: *const Plan, job: Plan.Job) ![]u8 { - const block_size = loader.plan_config.block_size; - const bytes = try allocator.alloc(u8, job.blockLen(block_size) * block_size); - errdefer allocator.free(bytes); - @memset(bytes, 0xaa); - const land = switch (job.kind) { - .dma_read => |read| { - // The end of the file cuts a widened read's padding. - var buffers = [_][]u8{bytes[0..read.len]}; - try readSlot(loader, plan.source_slot, &buffers, job.file_offset, read.minimum_len); - return bytes; - }, - .dma_land => |land| land, - .dma_fill, .host_direct, .host_bounce, .host_fill => unreachable, - }; - // A bounce buffer as large as any of the job's reads. - var longest: usize = land.len; - for (plan.extents[land.extent_start..][0..land.extent_len]) |extent| longest = @max(longest, extent.len); - const blocks = try allocator.alloc(host_memory.Block, job.blockLen(block_size)); - defer allocator.free(blocks); - for (blocks, 0..) |*block, index| block.* = bytes[index * block_size ..][0..block_size]; - var scratch: ReadRequest.Scratch = try .init(allocator, Planner.blockCount(longest, block_size), block_size); - defer scratch.deinit(); - try landJob(loader, plan, land, blocks, &scratch); - return bytes; - } - - /// One destination's device buffers, written by replayed transfers. - const Replay = struct { - shape: Shape, - placement: Sharding.Placement, - writer_size: usize, - bytes: []u8, - written: []bool, - - fn init(allocator: std.mem.Allocator, placed_shape: Shape) !Replay { - const placement: Sharding.Placement = try .init(&placed_shape); - const writers = placement.mesh().?.devicesInCanonicalOrder().len; - const writer_size = placement.localShape(placed_shape).byteSize(); - const bytes = try allocator.alloc(u8, writers * writer_size); - errdefer allocator.free(bytes); - @memset(bytes, 0xaa); - const written = try allocator.alloc(bool, bytes.len); - @memset(written, false); - return .{ .shape = placed_shape, .placement = placement, .writer_size = writer_size, .bytes = bytes, .written = written }; - } - - fn deinit(self: *Replay, allocator: std.mem.Allocator) void { - allocator.free(self.written); - allocator.free(self.bytes); - } - - /// Each byte of each device at most once. - fn write(self: *Replay, writer_mask: u64, destination_offset: usize, source: []const u8) !void { - var mask = writer_mask; - while (mask != 0) { - const writer: usize = @intCast(@ctz(mask)); - mask &= mask - 1; - try std.testing.expect(destination_offset + source.len <= self.writer_size); - const at = writer * self.writer_size + destination_offset; - for (self.written[at..][0..source.len]) |*flag| { - try std.testing.expect(!flag.*); - flag.* = true; - } - @memcpy(self.bytes[at..][0..source.len], source); - } - } - - /// Every device holds its slice of the rank-2 `global`, entirely. - fn expect(self: *const Replay, allocator: std.mem.Allocator, global: []const u8) !void { - for (self.written) |flag| try std.testing.expect(flag); - const shape = self.shape; - const element_bytes = shape.dtype().sizeOf(); - const row_bytes: usize = @intCast(shape.dim(1) * element_bytes); - const expected = try allocator.alloc(u8, self.writer_size); - defer allocator.free(expected); - for (0..self.placement.mesh().?.devicesInCanonicalOrder().len) |writer| { - const slices = self.placement.slices(writer, shape); - const rows = slices.get(0); - const columns = slices.get(1); - const len: usize = @intCast(columns.size * element_bytes); - var cursor: usize = 0; - for (@intCast(rows.start)..@intCast(rows.start + rows.size)) |row| { - @memcpy(expected[cursor..][0..len], global[row * row_bytes + @as(usize, @intCast(columns.start * element_bytes)) ..][0..len]); - cursor += len; - } - try std.testing.expectEqualSlices(u8, expected, self.bytes[writer * self.writer_size ..][0..self.writer_size]); - } - } - }; -}; - -test "composite destinations place each part and zero-fill the rest of every device" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const shape = Shape.init(.{ .row = 16, .col = 8 }, .u16); - // Two by two devices, then two by four. - for ([_]u32{ 4, 8 }) |device_count| { - var platform = Platform.init(allocator, io, .cpu, .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = device_count }, - }) catch return error.SkipZigTest; - defer platform.deinit(allocator, io); - const physical = &platform.physical_mesh; - const logical: Sharding.LogicalMesh = .mesh(.{ .model = .high_bandwidth }); - const halves: Sharding.Mesh = try .init("halves", physical, logical, .parseBindings(.{ .model = .link_x })); - const quarters: Sharding.Mesh = if (device_count == 4) - try .init("quarters", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })) - else - try .init("quarters", physical, logical, .parseBindings(.{ .model = .link_y })); - const all: Sharding.Mesh = try .init("all", physical, logical, .parseBindings(.{ .model = .{ .link_x, .link_y } })); - const scenarios = .{ - // Two row shards, each on half the devices: b and c straddle no - // boundary, the gap does. - .{ &halves, .{ .row = .model }, true }, - // Four row shards: b, c and the gap straddle shards. - .{ &quarters, .{ .row = .model }, true }, - // Every row of every source split across two devices, then - // across every device. - .{ &halves, .{ .col = .model }, false }, - .{ &all, .{ .col = .model }, false }, - // Every byte on every device: one span, so a and b merge. - .{ &all, .{}, true }, - }; - inline for (scenarios) |scenario| { - const placed = shape.withPartitioning(scenario[0], scenario[1]); - // 64-byte blocks hold a's tail and b's head together; 16-byte ones - // cut every row apart and every fill piece to one block. - try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .cross_part_piece = scenario[2] }); - try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .cross_part_piece = false }); - // Staged wherever it saves a submission: the column-sharded jobs - // land grouped by device, in reads exact or widened. - for ([_]usize{ 0, 8 }) |alignment| { - try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .alignment = alignment, .staging_gain = 1, .cross_part_piece = scenario[2] }); - try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 48, .alignment = alignment, .staging_gain = 1, .cross_part_piece = false }); - } - // Host jobs, read in place or through blocks when the reads are - // widened, are cut by blocks the same way. - for ([_]usize{ 0, 8 }) |alignment| { - try CompositeTest.run(platform, placed, .{ .block_size = 64, .request_size = 128, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = scenario[2] }); - try CompositeTest.run(platform, placed, .{ .block_size = 16, .request_size = 32, .memory = .host_pinned, .alignment = alignment, .cross_part_piece = false }); - } - } - } -} - -// Scheduler tests. - -/// A plan of `job_count` unit jobs (`file_offset` = index), each with one -/// block and one event slot, and no transfers, in `arena`. -fn testPlan(arena: std.mem.Allocator, job_count: usize) !*Plan { - const jobs = try arena.alloc(Plan.Job, job_count); - const requests = try arena.alloc(ReadRequest, job_count); - @memset(requests, ReadRequest.idle); - const events = try arena.alloc(Dma.EventContext, job_count); - for (jobs, 0..) |*job, index| job.* = .{ - .file_offset = index, - .transfer_start = 0, - .transfer_len = 0, - .kind = .{ .dma_read = .{ .len = 1, .minimum_len = 1 } }, - }; - const plan = try arena.create(Plan); - plan.* = .{ - .source_slot = undefined, - .jobs = jobs, - .transfers = &.{}, - .requests = requests, - .events = events, - .source_bytes = job_count, - }; - return plan; -} - -/// Publishes a plan of `job_count` unit jobs into an open batch. -fn publishTestPlan(scheduler: *Scheduler, batch: *Batch, job_count: usize) !void { - try scheduler.publish(std.testing.io, batch, try testPlan(batch.arena.allocator(), job_count)); -} - -/// A one-file submission's sequence: publish, then drop the sentinel. -fn publishTestBatch(scheduler: *Scheduler, job_count: usize) !*Batch { - const batch = try Batch.create(std.testing.allocator, std.testing.io, 0); - errdefer batch.destroy(); - try publishTestPlan(scheduler, batch, job_count); - batch.releaseUnits(1); - return batch; -} - -/// The jobs of the queued plans no worker claimed yet. -fn unclaimedJobs(scheduler: *const Scheduler) usize { - var jobs: usize = 0; - for (0..scheduler.queue.len) |index| { - const plan = scheduler.queue.at(index); - jobs += plan.jobs.len - plan.cursor; - } - return jobs; -} - -test "scheduler publishes a submission one file at a time and claims the files in order" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - - // Submitted out of file order: the sort groups "a" before "b". - var sources = [_]safetensors.Tensor{ - .{ .file_uri = "b", .name = "b0", .shape = .init(.{4}, .u8), .offset = 0 }, - .{ .file_uri = "a", .name = "a1", .shape = .init(.{4}, .u8), .offset = 4 }, - .{ .file_uri = "a", .name = "a0", .shape = .init(.{4}, .u8), .offset = 0 }, - }; - var slots = [_]SourceSlot{ - .{ .uri = "a" }, - .{ .uri = "b" }, - }; - var outputs: [sources.len]Buffer = undefined; - var destinations: [sources.len]Destination = undefined; - var parts: [sources.len]Part = undefined; - initPlainParts(platform, &sources, &.{ &slots[1], &slots[0], &slots[0] }, &outputs, &destinations, &parts); - - var scheduler: Scheduler = .init(allocator); - defer scheduler.deinit(); - const batch = try Batch.create(allocator, io, 0); - try Planner.publishFiles(&scheduler, io, batch, &parts, .{ - .block_size = 4, - .request_size = 4, - }, fixedReads(0)); - try std.testing.expectEqual(@as(usize, 2), batch.plans.items.len); - try std.testing.expectEqual(@as(usize, 2), batch.plans.items[0].jobs.len); - try std.testing.expectEqual(@as(usize, 1), batch.plans.items[1].jobs.len); - try std.testing.expectEqual(@as(usize, 3), unclaimedJobs(&scheduler)); - try std.testing.expect(batch.published_at != null); - - // File a's jobs in offset order, then file b's. - var claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan.source_slot == &slots[0]); - try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); - claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan.source_slot == &slots[0]); - try std.testing.expectEqual(@as(u64, 4), claim.plan.jobs[claim.index].file_offset); - claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan.source_slot == &slots[1]); - try std.testing.expectEqual(@as(u64, 0), claim.plan.jobs[claim.index].file_offset); - // Both plans left the queue with their last claim; the submission is - // still open until its sentinel goes. - try std.testing.expect(scheduler.claim(io) == null); - try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); - batch.releaseUnits(1); - try std.testing.expect(!batch.done.isSet()); - batch.releaseUnits(3); - try std.testing.expect(batch.done.isSet()); - batch.destroy(); -} - -test "fifo scheduler claims plan by plan in publish order, and a done batch leaves while a later plan is queued" { - const io = std.testing.io; - var scheduler: Scheduler = .init(std.testing.allocator); - defer scheduler.deinit(); - // An open submission's first plan, a sealed batch, then the open - // submission's second plan: an open submission holds no place of its - // own. - const open = try Batch.create(std.testing.allocator, io, 0); - try publishTestPlan(&scheduler, open, 1); - const sealed = try publishTestBatch(&scheduler, 2); - try publishTestPlan(&scheduler, open, 1); - try std.testing.expectEqual(@as(usize, 4), unclaimedJobs(&scheduler)); - - var claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan == open.plans.items[0] and claim.index == 0); - for (0..2) |index| { - claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan == sealed.plans.items[0] and claim.index == index); - } - // The sealed batch left the queue with its last claim, so it completes - // and goes away while the open one's second plan is still queued. - sealed.releaseUnits(2); - try std.testing.expect(sealed.done.isSet()); - sealed.destroy(); - try std.testing.expectEqual(@as(usize, 1), scheduler.queue.len); - - claim = scheduler.claim(io).?; - try std.testing.expect(claim.plan == open.plans.items[1] and claim.index == 0); - try std.testing.expect(scheduler.claim(io) == null); - try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); - // Claimed out, the open submission completes only once its sentinel - // goes. - open.releaseUnits(2); - try std.testing.expect(!open.done.isSet()); - open.releaseUnits(1); - try std.testing.expect(open.done.isSet()); - open.destroy(); -} - -test "fifo scheduler failure retires the unclaimed jobs of every queued plan and refuses later publishes" { - const io = std.testing.io; - var scheduler: Scheduler = .init(std.testing.allocator); - defer scheduler.deinit(); - // A batch without jobs is never queued and completes at publish. - const empty = try publishTestBatch(&scheduler, 0); - try std.testing.expect(empty.done.isSet()); - empty.destroy(); - // An open submission of two plans, then a sealed batch. - const open = try Batch.create(std.testing.allocator, io, 0); - try publishTestPlan(&scheduler, open, 2); - try publishTestPlan(&scheduler, open, 3); - const sealed = try publishTestBatch(&scheduler, 3); - try std.testing.expectEqual(@as(usize, 8), unclaimedJobs(&scheduler)); - try std.testing.expectEqual(@as(usize, 6), open.remaining.load(.acquire)); - - // The claim keeps its unit and the open submission its sentinel; every - // other job is retired. - try std.testing.expect(scheduler.claim(io).?.plan.batch == open); - scheduler.fail(io); - try std.testing.expectEqual(@as(usize, 0), scheduler.queue.len); - try std.testing.expectEqual(@as(usize, 2), open.remaining.load(.acquire)); - try std.testing.expect(sealed.done.isSet()); - sealed.destroy(); - try std.testing.expect(scheduler.claim(io) == null); - try std.testing.expect(!scheduler.waitForWork(io)); - // The submission goes on: its next plan is refused, and the sentinel - // drop leaves only the claim to complete it. - try std.testing.expectError(error.Canceled, publishTestPlan(&scheduler, open, 1)); - open.releaseUnits(1); - try std.testing.expect(!open.done.isSet()); - open.releaseUnits(1); - try std.testing.expect(open.done.isSet()); - open.destroy(); -} - -test "fifo scheduler wakes waiting workers on publish and releases them on failure" { - const WaitResult = enum(u8) { waiting, work, stopped }; - const io = std.testing.io; - var scheduler: Scheduler = .init(std.testing.allocator); - defer scheduler.deinit(); - - var woke: std.atomic.Value(WaitResult) = .init(.waiting); - var group: std.Io.Group = .init; - const Waiter = struct { - fn run(scheduler_: *Scheduler, io_: std.Io, woke_: *std.atomic.Value(WaitResult)) void { - woke_.store(if (scheduler_.waitForWork(io_)) .work else .stopped, .release); - } - }; - try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); - try io.sleep(.fromMilliseconds(5), .awake); - try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); - const batch = try publishTestBatch(&scheduler, 1); - try group.await(io); - try std.testing.expectEqual(WaitResult.work, woke.load(.acquire)); - _ = scheduler.claim(io).?; - batch.releaseUnits(1); - batch.destroy(); - - woke.store(.waiting, .release); - try group.concurrent(io, Waiter.run, .{ &scheduler, io, &woke }); - try io.sleep(.fromMilliseconds(5), .awake); - try std.testing.expectEqual(WaitResult.waiting, woke.load(.acquire)); - scheduler.fail(io); - try group.await(io); - try std.testing.expectEqual(WaitResult.stopped, woke.load(.acquire)); -} - -test "fifo scheduler concurrent claims across two batches return each job once" { - const io = std.testing.io; - var scheduler: Scheduler = .init(std.testing.allocator); - defer scheduler.deinit(); - const batches = [_]*Batch{ - try publishTestBatch(&scheduler, 16), - try publishTestBatch(&scheduler, 16), - }; - var seen: std.atomic.Value(u64) = .init(0); - var claim_count: std.atomic.Value(usize) = .init(0); - var duplicate: std.atomic.Value(bool) = .init(false); - var group: std.Io.Group = .init; - for (0..8) |_| try group.concurrent(io, struct { - fn run( - scheduler_: *Scheduler, - batches_: []const *Batch, - seen_: *std.atomic.Value(u64), - claim_count_: *std.atomic.Value(usize), - duplicate_: *std.atomic.Value(bool), - ) void { - while (scheduler_.claim(std.testing.io)) |claim| { - const base: u64 = if (claim.plan.batch == batches_[0]) 0 else 16; - const mask = @as(u64, 1) << @intCast(base + claim.plan.jobs[claim.index].file_offset); - if (seen_.fetchOr(mask, .acq_rel) & mask != 0) duplicate_.store(true, .release); - _ = claim_count_.fetchAdd(1, .monotonic); - } - } - }.run, .{ &scheduler, &batches, &seen, &claim_count, &duplicate }); - try group.await(io); - try std.testing.expectEqual(std.math.maxInt(u32), @as(u32, @truncate(seen.load(.acquire)))); - try std.testing.expectEqual(@as(usize, 32), claim_count.load(.acquire)); - try std.testing.expect(!duplicate.load(.acquire)); - try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&scheduler)); - for (batches) |batch| { - batch.releaseUnits(16); - try std.testing.expect(batch.done.isSet()); - batch.destroy(); - } -} - -// DMA pump and transfer target tests. - -test "device pump admits by bytes and by pieces" { - var pump: Dma.DevicePump = .{}; - try std.testing.expect(pump.hasRoom(8)); - // The piece that crosses the budget was admitted; the next is not. - pump.active_bytes = 8; - try std.testing.expect(!pump.hasRoom(8)); - pump.active_bytes = 7; - try std.testing.expect(pump.hasRoom(8)); - pump.active_pieces = max_dma_pieces_per_device; - try std.testing.expect(!pump.hasRoom(8)); -} - -test "the submission that completes a target's bytes carries the last flag" { - var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 100 }; - try std.testing.expect(!target.fullySubmitted()); - // Pieces arrive in any order; only the byte total matters. - try std.testing.expect(!target.nextIsLast(20)); - target.noteSubmitted(20); - try std.testing.expect(!target.nextIsLast(30)); - target.noteSubmitted(30); - try std.testing.expect(!target.nextIsLast(10)); - try std.testing.expect(target.nextIsLast(50)); - try std.testing.expect(!target.fullySubmitted()); - target.noteSubmitted(50); - try std.testing.expect(target.fullySubmitted()); -} - -// Request, credit and batch lifecycle tests. - -/// A single-device loader without PJRT, workers or pinned arenas: enough -/// for request, credit and batch lifecycle tests, with 64-byte blocks cut -/// as `create` cuts them, the zero block then `slots` credits of one block. -/// `platform` and `workspace` are its only undefined fields: what these -/// tests run reaches the platform only through a non-null PJRT event -/// (`Dma.EventContext.destroyEvent`). Must not move after `init`. -const TestLoader = struct { - memory: []u8, - pumps: [1]Dma.DevicePump = .{.{}}, - loader: Loader, - - fn init(self: *TestLoader, slots: usize) !void { - const allocator = std.testing.allocator; - const memory = try allocator.alloc(u8, (slots + 1) * 64); - errdefer allocator.free(memory); - const blocks = try allocator.alloc(host_memory.Block, slots + 1); - errdefer allocator.free(blocks); - for (blocks, 0..) |*block, index| block.* = memory[index * 64 ..][0..64]; - @memset(blocks[0], 0); - self.* = .{ .memory = memory, .loader = undefined }; - self.loader = .{ - .allocator = allocator, - .io = std.testing.io, - .platform = undefined, - .workspace = undefined, - .blocks = blocks, - .scheduler = .init(allocator), - .read_permits = .{ .permits = 1 }, - .read_width = 1, - .credits = try .init(allocator, blocks[1..], 1), - .pumps = &self.pumps, - .dma_budget_bytes = 64, - .created_at = .now(std.testing.io, .awake), - .plan_config = .{ .block_size = 64, .request_size = 64 }, - .reads = fixedReads(0), - }; - } - - fn deinit(self: *TestLoader) void { - const allocator = std.testing.allocator; - self.loader.scheduler.deinit(); - self.pumps[0].queue.deinit(allocator); - self.loader.credits.deinit(allocator); - allocator.free(self.loader.blocks); - allocator.free(self.memory); - } -}; - -test "a callback failing while a piece is queued drains it and completes the batch" { - const io = std.testing.io; - const allocator = std.testing.allocator; - var fixture: TestLoader = undefined; - try fixture.init(1); - defer fixture.deinit(); - const loader = &fixture.loader; - - // One request whose only piece is queued behind a DMA slot that a failed - // callback is about to free. - const batch = try publishTestBatch(&loader.scheduler, 1); - const request = loader.claimRequest().?; - var target: DestinationState.Target = .{ .manager = undefined, .device_index = 0, .total = 64 }; - _ = request.pending.fetchAdd(1, .acq_rel); - try fixture.pumps[0].queue.pushBack(allocator, .{ - .target = &target, - .request = request, - .data = loader.credits.slotBlocks(request.slot)[0], - .destination_offset = 0, - }); - fixture.pumps[0].active_bytes = 64; - fixture.pumps[0].active_pieces = 1; - request.release(); - try std.testing.expect(!batch.done.isSet()); - try std.testing.expectEqual(@as(usize, 1), loader.credits.inUse(io)); - _ = loader.first_error.set(error.Unknown); - - Dma.eventCompleted(loader, 0, 64); - try std.testing.expectEqual(@as(usize, 0), fixture.pumps[0].active_pieces); - try std.testing.expectEqual(@as(usize, 0), request.pending.load(.acquire)); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - try std.testing.expect(batch.done.isSet()); - - Dma.retireBatch(loader, batch); - batch.destroy(); -} - -test "batch completes when every claimed request completes" { - const io = std.testing.io; - var fixture: TestLoader = undefined; - try fixture.init(3); - defer fixture.deinit(); - const loader = &fixture.loader; - - const batch = try publishTestBatch(&loader.scheduler, 3); - try std.testing.expectEqual(@as(usize, 3), batch.remaining.load(.acquire)); - var requests: [3]*ReadRequest = undefined; - for (&requests) |*request| request.* = loader.claimRequest().?; - try std.testing.expect(loader.scheduler.claim(io) == null); - try std.testing.expectEqual(@as(usize, 3), loader.credits.inUse(io)); - - for (requests, 0..) |request, index| { - try std.testing.expect(!batch.done.isSet()); - try std.testing.expectEqual(requests.len - index, batch.remaining.load(.acquire)); - request.release(); - } - try std.testing.expect(batch.done.isSet()); - try std.testing.expectEqual(@as(usize, 0), batch.remaining.load(.acquire)); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - // Each claim took its job's slot of the plan's request array. - for (requests, batch.plans.items[0].requests) |request, *slot| { - try std.testing.expect(request == slot); - } - - Dma.retireBatch(loader, batch); - batch.destroy(); -} - -test "a worker waiting for a credit when the loader fails gets one back and stops" { - const io = std.testing.io; - var fixture: TestLoader = undefined; - try fixture.init(1); - defer fixture.deinit(); - const loader = &fixture.loader; - - // A request in flight holds the only credit; a worker that saw the - // second job waits on the credits. - const batch = try publishTestBatch(&loader.scheduler, 2); - const request = loader.claimRequest().?; - try loader.worker_group.concurrent(io, Loader.workerMain, .{loader}); - while (loader.credits.condition.state.load(.monotonic).waiters == 0) io.sleep(.fromMilliseconds(1), .awake) catch {}; - - // The failure retires the second job and closes nothing: the request - // gives its credit back as it ends, the worker claims nothing with it, - // and `destroy`'s shutdown stops it. - loader.recordError(error.InputOutput, "injected source failure", .{}); - try std.testing.expect(!batch.done.isSet()); - request.release(); - loader.stopWorkers(); - try std.testing.expect(batch.done.isSet()); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - try std.testing.expectEqual(@as(usize, 0), batch.plans.items[0].requests[1].pending.load(.acquire)); - Dma.retireBatch(loader, batch); - batch.destroy(); -} - -test "retired events are destroyed by the next pump or by a batch's retirement" { - // Without PJRT, contexts with a null event stand in for destroyed ones; - // the list mechanics are the subject. - const io = std.testing.io; - var fixture: TestLoader = undefined; - try fixture.init(2); - defer fixture.deinit(); - const loader = &fixture.loader; - - const batch = try publishTestBatch(&loader.scheduler, 2); - const requests = [_]*ReadRequest{ - loader.claimRequest().?, - loader.claimRequest().?, - }; - const plan = batch.plans.items[0]; - plan.events_used.store(2, .release); - for (plan.events, requests) |*ctx, request| ctx.* = .{ - .request = request, - .pjrt_event = null, - .device_index = 0, - .len = 64, - }; - // Two callbacks fired: the next pump destroys both, newest first. - Dma.retireEvent(loader, &plan.events[0]); - Dma.retireEvent(loader, &plan.events[1]); - const device_pump = &fixture.pumps[0]; - try std.testing.expect(device_pump.retired == &plan.events[1]); - try std.testing.expect(plan.events[1].next_retired == &plan.events[0]); - device_pump.mutex.lockUncancelable(io); - device_pump.destroyRetired(); - device_pump.mutex.unlock(io); - try std.testing.expect(device_pump.retired == null); - try std.testing.expect(plan.events[0].next_retired == null); - try std.testing.expect(plan.events[1].next_retired == null); - - // A callback that fires after the last pump leaves its context linked; - // the batch's retirement destroys it before the batch is freed, with - // the retired contexts of a batch still in flight. - var other: Dma.EventContext = .{ .request = undefined, .pjrt_event = null, .device_index = 0, .len = 64 }; - Dma.retireEvent(loader, &other); - Dma.retireEvent(loader, &plan.events[0]); - for (requests) |request| request.release(); - try std.testing.expect(batch.done.isSet()); - Dma.retireBatch(loader, batch); - try std.testing.expect(device_pump.retired == null); - try std.testing.expect(plan.events[0].next_retired == null); - try std.testing.expect(other.next_retired == null); - batch.destroy(); -} - -test "overlapping batches complete under concurrent claims and retirement" { - const io = std.testing.io; - const worker_count = 4; - var fixture: TestLoader = undefined; - try fixture.init(worker_count); - defer fixture.deinit(); - const loader = &fixture.loader; - - const Worker = struct { - fn run(loader_: *Loader) void { - while (loader_.scheduler.waitForWork(loader_.io)) if (loader_.claimRequest()) |request| request.release(); - } - }; - var group: std.Io.Group = .init; - defer { - loader.scheduler.fail(io); - group.await(io) catch {}; - } - for (0..worker_count) |_| try group.concurrent(io, Worker.run, .{loader}); - - // Three batches in flight at a time, awaited newest first: a batch is - // retired while later ones are still being claimed. - for (0..70) |round| { - var batches: [3]*Batch = undefined; - var job_counts: [3]usize = undefined; - for (&batches, &job_counts, 0..) |*batch, *job_count, offset| { - job_count.* = (round * 3 + offset) % 7; - batch.* = try publishTestBatch(&loader.scheduler, job_count.*); - } - var index = batches.len; - while (index > 0) { - index -= 1; - const batch = batches[index]; - batch.done.waitUncancelable(io); - Dma.retireBatch(loader, batch); - batch.destroy(); - } - try std.testing.expectEqual(@as(usize, 0), unclaimedJobs(&loader.scheduler)); - } - // Every request released its credit before its batch completed; the - // workers only hold one while a claim is in progress. - loader.scheduler.fail(io); - try group.await(io); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); -} - -// Source read tests. - -test "readFilePositionalAllV resumes short reads and stops at minimum" { - const OneByteIo = struct { - const file_end = 11; - - /// Reads one byte per call, the low byte of its offset, from a file - /// that ends at `file_end`, and counts the calls in `userdata`. - fn fileReadPositional(userdata: ?*anyopaque, _: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const calls: *usize = @ptrCast(@alignCast(userdata.?)); - calls.* += 1; - if (offset >= file_end) return 0; - for (data) |buffer| { - if (buffer.len == 0) continue; - buffer[0] = @truncate(offset); - return 1; - } - return 0; - } - }; - var vtable = std.testing.io.vtable.*; - vtable.fileReadPositional = OneByteIo.fileReadPositional; - var calls: usize = 0; - const io: std.Io = .{ .userdata = &calls, .vtable = &vtable }; - const file: std.Io.File = .{ .handle = 0, .flags = .{ .nonblocking = false } }; - - var output: [5]u8 = @splat(0xaa); - var buffers = [_][]u8{ output[0..1], output[1..1], output[1..3], output[3..5] }; - try std.testing.expectEqual(@as(u64, 4), try readFilePositionalAllV(io, file, &buffers, 7, 4)); - try std.testing.expectEqual(@as(usize, 4), calls); - try std.testing.expectEqualSlices(u8, &.{ 7, 8, 9, 10, 0xaa }, &output); - - var past_end = [_][]u8{&output}; - try std.testing.expectError(error.UnexpectedEndOfFile, readFilePositionalAllV(io, file, &past_end, 7, output.len)); -} - -// LazyOnce tests. - -test "lazy initialization preserves its finite error across repeated callers" { - const Context = struct { - fn init(calls: *usize) error{FileNotFound}!u32 { - calls.* += 1; - return error.FileNotFound; - } - }; - var once: LazyOnce(u32) = .{}; - var calls: usize = 0; - try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); - try std.testing.expectError(error.FileNotFound, once.ensure(std.testing.io, Context.init, .{&calls})); - try std.testing.expectEqual(1, calls); -} - -// Whole-loader tests. - -/// Submits `specs`, which must fail with `expected`; a batch that went out -/// anyway is awaited before the test fails. -fn expectSubmitError(loader: *Loader, specs: []const LoadSpec, expected: anyerror) !void { - const batch = loader.submit(specs, .none) catch |err| return std.testing.expectEqual(expected, err); - loader.awaitBatch(batch) catch {}; - return error.TestUnexpectedResult; -} - -test "loader releases the pinned set when its workers cannot start" { - const allocator = std.testing.allocator; - const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, std.testing.io); - // The testing io, but no worker starts: `create` fails after the carve, - // the credits and the pumps, and unwinds every one. - var vtable = std.testing.io.vtable.*; - vtable.groupConcurrent = std.Io.failingGroupConcurrent; - const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; - const result: anyerror!void = if (Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off })) |loader| unexpected: { - loader.destroy(); - break :unexpected {}; - } else |err| err; - try std.testing.expectError(error.ConcurrencyUnavailable, result); -} - -test "loader failures clean up before publication, after publication and during reading" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const contents = [_]u8{ 1, 2, 3, 4 }; - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); - var later_buffer: [1024]u8 = undefined; - const later_path = try writeTestFile(tmp.dir, io, "weights.bin.later", &contents, &later_buffer); - const sources = [_]safetensors.Tensor{ - .{ .file_uri = path, .name = "first", .shape = .init(.{4}, .u8), .offset = 0 }, - .{ .file_uri = later_path, .name = "later", .shape = .init(.{5}, .u8), .offset = 0 }, - }; - const parts = [_]LoadPart{ .whole(&sources[0]), .whole(&sources[1]) }; - - // Device memory goes through blocks and DMA; host memory is written in - // place by the workers, into views of it for `host_mapped`. All clean - // up the same way. - for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { - // Planning allocates from the scheduler's allocator, whose - // allocations fail in turn: the first file's planning fails before - // any plan is published, then the later file's after the first - // file's plan was. Past the last, planning succeeds and the read - // fails: the later source ends after four of five bytes. - var fail_index: usize = 0; - while (true) : (fail_index += 1) { - var outputs: [2]Buffer = @splat(.{ - ._platform = platform, - ._shape = sources[0].shape, - ._shards = .empty, - }); - defer for (&outputs) |*output| output.deinit(); - var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = fail_index }); - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - loader.scheduler.allocator = failing.allocator(); - const specs = [_]LoadSpec{ - .{ .shape = sources[0].shape, .memory = memory, .output = &outputs[0], .parts = parts[0..1] }, - .{ .shape = sources[1].shape, .memory = memory, .output = &outputs[1], .parts = parts[1..2] }, - }; - - if (loader.submit(&specs, .none)) |batch| { - try std.testing.expectError(error.UnexpectedEndOfFile, loader.awaitBatch(batch)); - } else |err| try std.testing.expectEqual(error.OutOfMemory, err); - try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); - // A worker the batch woke may still hold the credit it took for - // a claim that finds nothing: every credit is free once the - // workers stopped (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); - - // Each failure fails the loader: a later submission gets it. - try std.testing.expectEqual(@as(usize, 1), loader.batch_count); - const expected: anyerror = if (failing.has_induced_failure) error.OutOfMemory else error.UnexpectedEndOfFile; - try std.testing.expectError(expected, loader.checkOpen()); - try expectSubmitError(loader, specs[0..1], expected); - if (!failing.has_induced_failure) break; - } - } -} - -test "exactly the width of reads are in flight, and a read failing mid-load leaves nothing held" { - const allocator = std.testing.allocator; - const platform = Platform.init(allocator, std.testing.io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, std.testing.io); - // Every source read holds for a moment and is counted; the one numbered - // `fail_at` fails. - const Reads = struct { - var in_flight: std.atomic.Value(usize) = .init(0); - var peak: std.atomic.Value(usize) = .init(0); - var started: std.atomic.Value(usize) = .init(0); - var fail_at: usize = std.math.maxInt(usize); - - fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - _ = peak.fetchMax(in_flight.fetchAdd(1, .acq_rel) + 1, .acq_rel); - defer _ = in_flight.fetchSub(1, .acq_rel); - std.testing.io.sleep(.fromMilliseconds(5), .awake) catch {}; - if (started.fetchAdd(1, .acq_rel) == fail_at) return error.InputOutput; - return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); - } - }; - var vtable = std.testing.io.vtable.*; - vtable.fileReadPositional = Reads.readPositional; - const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; - - // Sixteen tensors with gaps between them: a read, and a job, each. - const tensors = 16; - var contents: [tensors * 8]u8 = undefined; - for (&contents, 0..) |*byte, index| byte.* = @truncate(index); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", &contents, &path_buffer); - var sources: [tensors]safetensors.Tensor = undefined; - var parts: [tensors]LoadPart = undefined; - var specs: [tensors]LoadSpec = undefined; - var outputs: [tensors]Buffer = undefined; - for (&sources, &parts, &specs, &outputs, 0..) |*source, *part, *spec, *output, index| { - source.* = .{ .file_uri = path, .name = "t", .shape = .init(.{4}, .u8), .offset = index * 8 }; - part.* = .whole(source); - spec.* = .{ .shape = source.shape, .output = output, .parts = part[0..1] }; - } - - for ([_]?usize{ null, 5 }) |fail_at| { - for (&outputs) |*output| output.* = .{ ._platform = platform, ._shape = sources[0].shape, ._shards = .empty }; - defer for (&outputs) |*output| output.deinit(); - // `destroy` returns: no worker waits for a credit or a permit. - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - try std.testing.expectEqual(@as(usize, 2), loader.read_width); - Reads.in_flight = .init(0); - Reads.peak = .init(0); - Reads.started = .init(0); - Reads.fail_at = fail_at orelse std.math.maxInt(usize); - const batch = try loader.submit(&specs, .none); - if (fail_at == null) { - try loader.awaitBatch(batch); - // Three workers, two permits. - try std.testing.expectEqual(@as(usize, tensors), Reads.started.load(.acquire)); - try std.testing.expectEqual(loader.read_width, Reads.peak.load(.acquire)); - try testing.expectShards(allocator, std.testing.io, &outputs[tensors - 1], contents[(tensors - 1) * 8 ..][0..4], .default); - } else { - try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); - // The reads stopped at the failure. - try std.testing.expect(Reads.started.load(.acquire) < tensors); - } - try std.testing.expectEqual(@as(usize, 0), loader.scheduler.queue.len); - // A worker the batch woke may still hold the credit it took for a - // claim that finds nothing: every credit is free once the workers - // stopped (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - try std.testing.expectEqual(loader.read_width, loader.read_permits.permits); - } -} - -test "a default-memory output a failed load left unfinished reports an error on every device" { - const allocator = std.testing.allocator; - // On the accelerator when it has two devices: each plugin's transfer - // manager errors the buffers it leaves unfinished. - var devices: testing.Devices = undefined; - try devices.init(allocator, std.testing.io, 2); - defer devices.deinit(); - const platform = devices.platform; - // The read reaching the last request fails. - const Reads = struct { - var fail_from: u64 = std.math.maxInt(u64); - - fn readPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - if (offset >= fail_from) return error.InputOutput; - return std.testing.io.vtable.fileReadPositional(userdata, file, data, offset); - } - }; - var vtable = std.testing.io.vtable.*; - vtable.fileReadPositional = Reads.readPositional; - const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - - // A replicated tensor of three requests, a job each: no device's last - // piece goes out. - const request_size = loader.plan_config.request_size; - const contents = try allocator.alloc(u8, 3 * request_size); - defer allocator.free(contents); - testing.pattern(contents, 1); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, std.testing.io, "weights.bin", contents, &path_buffer); - var source: safetensors.Tensor = .{ .file_uri = path, .name = "t", .shape = .init(.{contents.len}, .u8), .offset = 0 }; - var part: LoadPart = .whole(&source); - var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; - defer output.deinit(); - const spec: LoadSpec = .{ .shape = source.shape, .output = &output, .parts = (&part)[0..1] }; - - Reads.fail_from = 2 * request_size; - defer Reads.fail_from = std.math.maxInt(u64); - const batch = try loader.submit((&spec)[0..1], .none); - try std.testing.expectError(error.InputOutput, loader.awaitBatch(batch)); - // The failing job created the buffers before its read. Each one is - // ready, with an error, once the batch is retired: a success would - // hand out a buffer missing its last request, a pending one would hang - // its first user. - const placed = platform.placedShape(source.shape); - try std.testing.expectEqual(placed._sharding.mesh.?.devicesInCanonicalOrder().len, output._shards.len); - for (output._shards.constSlice()) |shard| { - const event = shard.readyEvent(platform.pjrt_api); - defer event.deinit(platform.pjrt_api); - try std.testing.expect(event.isReady(platform.pjrt_api)); - const err = event.getEventError(platform.pjrt_api) orelse return error.TestUnexpectedResult; - err.deinit(platform.pjrt_api); - } -} - -test "whole tensors load on four devices, sharded by rows or columns, replicated, or joined from two files" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 4); - defer devices.deinit(); - const mesh = try devices.mesh("tp"); - - // Rows of 32 bytes, 16 per device; rows cut into four pieces of 8 - // bytes, one per device; a table on every device; and a tensor whose - // rows continue in a second file. - var rows: [64 * 32]u8 = undefined; - var cols: [1024 * 32]u8 = undefined; - var table: [3000 * 4]u8 = undefined; - var tail: [64 * 32]u8 = undefined; - testing.pattern(&rows, 1); - testing.pattern(&cols, 2); - testing.pattern(&table, 3); - testing.pattern(&tail, 4); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ - .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, - .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, - .{ .name = "table", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &table }, - }); - try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ - .{ .name = "tail", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &tail }, - }); - var joined: [rows.len + tail.len]u8 = undefined; - @memcpy(joined[0..rows.len], &rows); - @memcpy(joined[rows.len..], &tail); - const parts = [_]LoadPart{ - .whole(registry.tensors.getPtr("rows").?), - .whole(registry.tensors.getPtr("cols").?), - .whole(registry.tensors.getPtr("table").?), - // rows again, then tail after it: the first file's run overlaps. - .whole(registry.tensors.getPtr("rows").?), - .{ .source = registry.tensors.getPtr("tail").?, .lattice = .{ .dst_offset = rows.len, .width = tail.len } }, - }; - var outputs: [4]Buffer = undefined; - const specs = [_]LoadSpec{ - .{ .shape = Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[0], .parts = parts[0..1] }, - .{ .shape = Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), .output = &outputs[1], .parts = parts[1..2] }, - .{ .shape = Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{}), .output = &outputs[2], .parts = parts[2..3] }, - .{ .shape = Shape.init(.{ .row = 128, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), .output = &outputs[3], .parts = parts[3..5] }, - }; - const globals = [_][]const u8{ &rows, &cols, &table, &joined }; - - // Blocks of 4 KiB and requests of two: the first file's run is cut into - // jobs, and the jobs' transfers at block ends. - const loader = try Loader.create(allocator, io, devices.platform, .{ - .read_parallelism = 2, - .read_hints = .{ .read_chunk_size = 8192 }, - .dma_block_size = 4096, - .direct_io = .off, - }); - defer loader.destroy(); - const batch = try loader.submit(&specs, .none); - var jobs: usize = 0; - for (batch.plans.items) |plan| jobs += plan.jobs.len; - try loader.awaitBatch(batch); - defer for (&outputs) |*output| output.deinit(); - for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); - // 46,816 bytes in the first file, 2,048 in the second. - try std.testing.expectEqual(@as(usize, 6 + 1), jobs); - // Every credit is free once the workers stopped (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); -} - -test "a submit the loader fails between its open check and its first publish returns that failure" { - const io = std.testing.io; - const platform = Platform.init(std.testing.allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(std.testing.allocator, io); - // Fails the loader in the submitting task's next allocation: after - // `submit`'s open check and before its first publish, where a worker of - // an older batch can fail it and the scheduler then answers `Canceled`. - const FailNextAllocation = struct { - parent: std.mem.Allocator, - loader: *Loader = undefined, - thread: std.Thread.Id = undefined, - armed: std.atomic.Value(bool) = .init(false), - - fn arm(self: *@This(), loader: *Loader) void { - self.loader = loader; - self.thread = std.Thread.getCurrentId(); - self.armed.store(true, .release); - } - - fn allocator(self: *@This()) std.mem.Allocator { - return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { - const self: *@This() = @ptrCast(@alignCast(ctx)); - // The workers allocate on threads of their own. - if (self.armed.load(.acquire) and self.thread == std.Thread.getCurrentId()) { - self.armed.store(false, .monotonic); - self.loader.recordError(error.UnexpectedEndOfFile, "an older batch's read", .{}); - } - return self.parent.rawAlloc(len, alignment, ret_addr); - } - - fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { - const self: *@This() = @ptrCast(@alignCast(ctx)); - return self.parent.rawResize(memory, alignment, new_len, ret_addr); - } - - fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { - const self: *@This() = @ptrCast(@alignCast(ctx)); - return self.parent.rawRemap(memory, alignment, new_len, ret_addr); - } - - fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { - const self: *@This() = @ptrCast(@alignCast(ctx)); - self.parent.rawFree(memory, alignment, ret_addr); - } - }; - var failing: FailNextAllocation = .{ .parent = std.testing.allocator }; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, io, "weights.bin", &.{ 1, 2, 3, 4 }, &path_buffer); - var source: safetensors.Tensor = .{ .file_uri = path, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; - var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; - defer output.deinit(); - const loader = try Loader.create(failing.allocator(), io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - const parts = [_]LoadPart{.whole(&source)}; - failing.arm(loader); - try expectSubmitError(loader, &.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, error.UnexpectedEndOfFile); -} - -test "source slots keep their own copy of each path" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const contents = [_]u8{ 1, 2, 3, 4 }; - var path_buffer: [1024]u8 = undefined; - const path = try writeTestFile(tmp.dir, io, "weights.bin", &contents, &path_buffer); - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - // Each submission names the file through a string of its own, freed - // right after the submit: workers open the file through its slot. - for (0..2) |_| { - const uri = try allocator.dupe(u8, path); - var source: safetensors.Tensor = .{ .file_uri = uri, .name = "x", .shape = .init(.{4}, .u8), .offset = 0 }; - var output: Buffer = .{ ._platform = platform, ._shape = source.shape, ._shards = .empty }; - defer output.deinit(); - const parts = [_]LoadPart{.whole(&source)}; - const submitted = loader.submit(&.{.{ .shape = source.shape, .output = &output, .parts = &parts }}, .none); - allocator.free(uri); - try loader.awaitBatch(try submitted); - try testing.expectShards(allocator, io, &output, &contents, .default); - } - // The testing allocator overwrites freed bytes: a slot keyed by the - // first string would miss the second lookup. - try std.testing.expectEqual(@as(usize, 1), loader.source_slots.count()); -} - -test "column-sharded tensors load as one DMA piece per device and block, read exact or widened" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 4); - defer devices.deinit(); - const mesh = try devices.mesh("tp"); - - // Rows of 32 bytes, 8 per device; then 12,000 one-byte pieces, more - // than one vectored read takes. - var rows: [64 * 32]u8 = undefined; - var cols: [1024 * 32]u8 = undefined; - var narrow: [3000 * 4]u8 = undefined; - testing.pattern(&rows, 1); - testing.pattern(&cols, 2); - testing.pattern(&narrow, 3); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "staged.safetensors", ®istry, &.{ - .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 16 }, .bytes = &rows }, - .{ .name = "cols", .dtype = "U16", .dims = &.{ 1024, 16 }, .bytes = &cols }, - .{ .name = "narrow", .dtype = "U8", .dims = &.{ 3000, 4 }, .bytes = &narrow }, - }); - const names = [_][]const u8{ "rows", "cols", "narrow" }; - const shapes = [_]Shape{ - Shape.init(.{ .row = 64, .col = 16 }, .u16).withPartitioning(&mesh, .{ .row = .model }), - Shape.init(.{ .row = 1024, .col = 16 }, .u16).withPartitioning(&mesh, .{ .col = .model }), - Shape.init(.{ .row = 3000, .col = 4 }, .u8).withPartitioning(&mesh, .{ .col = .model }), - }; - const globals = [_][]const u8{ &rows, &cols, &narrow }; - const row_pieces = 1024 * 4 + 3000 * 4; - - for ([_]usize{ 0, 64 }) |alignment| { - var parts: [names.len]LoadPart = undefined; - var specs: [names.len]LoadSpec = undefined; - var outputs: [names.len]Buffer = undefined; - for (&parts, &specs, &outputs, names, shapes) |*part, *spec, *output, name, shape| { - part.* = .whole(registry.tensors.getPtr(name).?); - spec.* = .{ .shape = shape, .output = output, .parts = part[0..1] }; - } - const loader = try Loader.create(allocator, io, devices.platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - loader.reads.alignment = alignment; - const batch = try loader.submit(&specs, .none); - // Every plan is published once `submit` returns. - var staged_jobs: usize = 0; - var submissions: usize = 0; - for (batch.plans.items) |plan| { - for (plan.jobs) |job| staged_jobs += @intFromBool(job.kind == .dma_land); - submissions += plan.events.len; - } - try loader.awaitBatch(batch); - defer for (&outputs) |*output| output.deinit(); - for (&outputs, globals) |*output, global| try testing.expectShards(allocator, io, output, global, .default); - try std.testing.expect(staged_jobs >= 1); - try std.testing.expect(submissions < row_pieces / 100); - // Every credit is free once the workers stopped (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - } -} - -test "a strided part of a remote file is checked against its exact source range" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 1); - defer devices.deinit(); - var bytes: [128 * 128]u8 = undefined; - testing.pattern(&bytes, 1); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "square.safetensors", ®istry, &.{ - .{ .name = "square", .dtype = "U8", .dims = &.{ 128, 128 }, .bytes = &bytes }, - }); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - // The same file through a registered scheme: a remote file, read exact. - try vfs.registerBackend("remote", .{ .io = io }); - const local = registry.tensors.get("square").?; - var remote = local; - var uri_buffer: [1100]u8 = undefined; - remote.file_uri = try std.fmt.bufPrint(&uri_buffer, "remote://{s}", .{local.file_uri}); - std.debug.assert(local.offset % 4096 != 0); - - var transposed: [128 * 128]u8 = undefined; - for (0..128) |column| { - for (0..128) |row| transposed[column * 128 + row] = bytes[row * 128 + column]; - } - // Transposed: its source read whole by every job, a request exactly, - // at an offset off the direct I/O alignment. - const dims = [_]backend.Lattice.Dim{ - .{ .count = 128, .src_stride = 1, .dst_stride = 128 }, - .{ .count = 128, .src_stride = 128, .dst_stride = 1 }, - }; - const loader = try Loader.create(allocator, vfs.io(), devices.platform, .{ .read_parallelism = 2 }); - defer loader.destroy(); - try std.testing.expectEqual(VFS.direct_io_alignment, loader.reads.alignment); - loader.plan_config.request_size = bytes.len; - - var output: Buffer = undefined; - const part: LoadPart = .{ .source = &remote, .lattice = .{ .width = 1, .dims = &dims } }; - const fit = loader.stridedRead(part); - try std.testing.expectEqual(fit.limit, fit.read); - const spec: LoadSpec = .{ .shape = .init(.{ 128, 128 }, .u8), .output = &output, .parts = (&part)[0..1] }; - try loader.awaitBatch(try loader.submit(&.{spec}, .none)); - defer output.deinit(); - try testing.expectShards(allocator, io, &output, &transposed, .default); - - // A local file may be read directly: its range widened to the alignment - // no longer fits. - if (VFS.direct_io_alignment != 0) { - const local_part: LoadPart = .{ .source = &local, .lattice = .{ .width = 1, .dims = &dims } }; - const widened = loader.stridedRead(local_part); - try std.testing.expect(widened.read > widened.limit); - } -} - -/// A host-resident load on four devices: rows sharded; columns sharded, two -/// bytes per device per row, so one job has 8,192 transfers and scatters in -/// several reads; replicated; one source twice, replicated and row-sharded, -/// which overlaps; and a device-memory tensor in the same file. -const HostResidentTest = struct { - /// Each run writes its own file from `seed`: a buffer that lands on - /// memory an earlier run freed must not find the expected bytes there. - fn run(devices: *const testing.Devices, seed: u8, alignment: usize) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = devices.platform; - const mesh = try devices.mesh("tp"); - - var rows: [64 * 16]u8 = undefined; - var dev: [16 * 16]u8 = undefined; - var cols: [2048 * 8]u8 = undefined; - var rep: [96 * 4]u8 = undefined; - var twice: [32 * 4]u8 = undefined; - testing.pattern(&rows, seed); - testing.pattern(&dev, seed +% 1); - testing.pattern(&cols, seed +% 2); - testing.pattern(&rep, seed +% 3); - testing.pattern(&twice, seed +% 4); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - // The device tensor separates the host runs: an overlap sends its - // whole job through blocks, so the twice-bound source must not share - // a job with the others. - try testing.writeSafetensors(allocator, io, tmp.dir, "host.safetensors", ®istry, &.{ - .{ .name = "rows", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &rows }, - .{ .name = "cols", .dtype = "U8", .dims = &.{ 2048, 8 }, .bytes = &cols }, - .{ .name = "rep", .dtype = "U8", .dims = &.{ 96, 4 }, .bytes = &rep }, - .{ .name = "dev", .dtype = "U16", .dims = &.{ 16, 8 }, .bytes = &dev }, - .{ .name = "twice", .dtype = "U8", .dims = &.{ 32, 4 }, .bytes = &twice }, - }); - - const names = [_][]const u8{ "rows", "dev", "cols", "rep", "twice", "twice" }; - const shapes = [_]Shape{ - Shape.init(.{ .row = 64, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), - Shape.init(.{ .row = 16, .col = 8 }, .u16).withPartitioning(&mesh, .{ .row = .model }), - Shape.init(.{ .row = 2048, .col = 8 }, .u8).withPartitioning(&mesh, .{ .col = .model }), - Shape.init(.{ .row = 96, .col = 4 }, .u8).withPartitioning(&mesh, .{}), - Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{}), - Shape.init(.{ .row = 32, .col = 4 }, .u8).withPartitioning(&mesh, .{ .row = .model }), - }; - // `rows` as views of pinned memory in the devices' default space. - const memories = [_]Memory{ .host_mapped, .default, .host_pinned, .host_pinned, .host_pinned, .host_pinned }; - const globals = [_][]const u8{ &rows, &dev, &cols, &rep, &twice, &twice }; - var parts: [names.len]LoadPart = undefined; - var specs: [names.len]LoadSpec = undefined; - var outputs: [names.len]Buffer = undefined; - for (&parts, &specs, &outputs, names, shapes, memories) |*part, *spec, *output, name, shape, memory| { - part.* = .whole(registry.tensors.getPtr(name).?); - spec.* = .{ .shape = shape, .memory = memory, .output = output, .parts = part[0..1] }; - } - - const loader = try Loader.create(allocator, io, platform, .{ .read_parallelism = 2, .direct_io = .off }); - defer loader.destroy(); - loader.reads.alignment = alignment; - const live_before = SharedHostAllocation.live.load(.acquire); - { - const batch = try loader.submit(&specs, .none); - try loader.awaitBatch(batch); - defer for (&outputs) |*output| output.deinit(); - // Host shards in each device's host memory, or its default - // memory for views. - for (&outputs, globals, memories) |*output, global, memory| try testing.expectShards(allocator, io, output, global, memory); - // Every credit, with its blocks, came back once the workers - // stopped (`stopWorkers`). - loader.stopWorkers(); - try std.testing.expectEqual(@as(usize, 0), loader.credits.inUse(io)); - - // The replicated host ones are one allocation. - for (outputs[3..5]) |output| { - const shards = output._shards.constSlice(); - const first = try shards[0].opaqueDeviceMemoryDataPointer(platform.pjrt_api); - for (shards[1..]) |shard| { - const pointer = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api); - try std.testing.expect(pointer == first); - } - } - // The two replicated outputs, and one per device of `rows`. - try std.testing.expectEqual(live_before + 2 + outputs[0]._shards.len, SharedHostAllocation.live.load(.acquire)); - // Shards may be freed in any order: free one replicated output's - // last first; its allocation goes with its last view. - std.mem.reverse(*pjrt.Buffer, outputs[3]._shards.slice()); - } - // PJRT may run the views' deletion callbacks on its own threads, - // just after the shards are freed. - var polls: usize = 0; - while (SharedHostAllocation.live.load(.acquire) != live_before) : (polls += 1) { - if (polls == 5000) return error.TestUnexpectedResult; - try io.sleep(.fromMilliseconds(1), .awake); - } - } -}; - -test "host-resident destinations are written in place, through blocks when reads are widened or overlap, and replicas share one allocation" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var devices: testing.Devices = undefined; - try devices.init(allocator, io, 4); - defer devices.deinit(); - if (!backend.supportsMemory(devices.platform, .host_pinned)) return error.SkipZigTest; - // In place, then through blocks for every host job. - for ([_]usize{ 0, 64 }, [_]u8{ 1, 5 }) |alignment, seed| { - try HostResidentTest.run(&devices, seed, alignment); - } -} diff --git a/zml/io/dma_calibration.zig b/zml/io/dma_calibration.zig deleted file mode 100644 index e744a9e6..00000000 --- a/zml/io/dma_calibration.zig +++ /dev/null @@ -1,675 +0,0 @@ -//! DMA sizing for the direct backend: the block size, decided once when the -//! backend is created. It screens `default_block_sizes` (2 to 32 MiB), or -//! the one block `zml.io.Loader.Options.dma_block_size` gives, on one -//! representative device, at `depth` transfers per device and over fixed -//! windows, and keeps the smallest block within 8% of the measured peak; -//! borderline candidates get longer, alternating confirmation windows. The -//! ring it measures in stays mapped as the load's first pinned capacity. -//! CPU returns the smallest candidate without measuring: its plugin's -//! transfer is a memcpy, and a load takes the same time at every block -//! size. The buffered backend has nothing to size. -//! -//! `measure` runs the same warm-up and screen without a loader, with every -//! knob, for load_check's `dma-bench` (examples/io/load_check.zig). -const std = @import("std"); - -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); - -const Platform = @import("../platform.zig").Platform; -const backend = @import("backend.zig"); -const host_memory = @import("host_memory.zig"); -const log = @import("log.zig").io; - -/// Blocks in flight per device: the screen runs `depth` transfers (the -/// default `Options.block_parallelism`), and the direct backend's DMA -/// budget is `depth` blocks per device, in bytes (`DevicePump.hasRoom` in -/// direct_loader.zig). Eight repeatedly won or tied on MI300X; wider stages -/// mostly increased callback latency and pinned memory, and an adaptive DMA -/// width added state with little load benefit. At 32, four GB300 loaded -/// DeepSeek 9% faster for 2.5x the pinned memory. -pub const depth: usize = 8; - -/// Screened, not fixed: eight MI300X loaded replicated Gemma in 7.83 s at -/// 16 MiB against 10.69 s at 8, whereas one B70 lost 10.5% at 16 MiB. -pub const default_block_sizes = [_]usize{ - 2 * 1024 * 1024, - 4 * 1024 * 1024, - 8 * 1024 * 1024, - 16 * 1024 * 1024, - 32 * 1024 * 1024, -}; - -/// The screen's knobs. The loader sets only `block_sizes` and keeps the -/// other defaults; `measure` takes them all. -pub const Options = struct { - block_sizes: []const usize = &default_block_sizes, - /// Transfers in flight per device in the screen, at most - /// `max_block_parallelism`. The loader keeps `depth`. - block_parallelism: usize = depth, - /// A screen window runs for at least this long and, unless the target is - /// zero, until the representative device completes the transfer target. - /// On eight MI300X, 2 ms/32-transfer windows calibrate in 0.96 s against - /// 4.83 s at 10 ms/128, selecting the same block. Short screens sometimes - /// selected the wrong block under noise, hence the longer borderline - /// confirmation windows (`confirmation_duration`). - minimum_duration: std.Io.Duration = .fromMilliseconds(2), - minimum_transfers: u64 = 32, - /// Prefer the smallest block within this tolerance of the measured peak - /// transfer throughput. - block_selection_tolerance: f64 = 0.08, -}; - -/// Borderline block candidates receive longer paired windows, alternating -/// which runs first. A candidate is borderline when its rounds disagree on -/// whether it is near the peak, or when its median ratio to the peak lies -/// within `confirmation_margin` of the tolerance's floor. -const confirmation_duration: std.Io.Duration = .fromMilliseconds(25); -const confirmation_minimum_transfers: u64 = 256; -const confirmation_margin: f64 = 0.02; - -/// The widest screen `measure` takes. -pub const max_block_parallelism: usize = 32; - -/// Measures as the direct loader's creation does, without a loader and with -/// every knob of `opts`, and returns the block it selects: warms the device -/// allocators, then screens the blocks in a workspace of its own, placed -/// near the devices as for reads through the page cache (it reads no file), -/// released before it returns. CPU returns the smallest block without -/// measuring; a platform that loads through the buffered backend -/// (`backend.loadsDirect`) returns `UnsupportedPlatform`, and a knob out of -/// range `InvalidOptions`. Every failure is logged. For load_check's -/// `dma-bench`. -pub fn measure(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) !usize { - try validate(opts); - if (!backend.loadsDirect(platform)) { - log.err("DMA calibration: {t} loads through the buffered backend, which has no DMA to measure: UnsupportedPlatform", .{platform.target}); - return error.UnsupportedPlatform; - } - platform.warmupDeviceAllocators(io) catch |err| { - log.failure(err, "DMA calibration: warm the device allocators: target={t}", .{platform.target}); - return err; - }; - var workspace = try host_memory.Workspace.init(allocator, io, platform, false); - defer workspace.deinit(); - return calibrate(&workspace, platform, opts) catch |err| { - log.failure(err, "DMA calibration: target={t}", .{platform.target}); - return err; - }; -} - -/// A block is at most the largest source request, the bound -/// `zml.io.Loader.init` puts on its own block sizes, so every -/// candidate's ring of `block_parallelism` blocks fits the workspace's -/// mapped ceiling. -fn validate(opts: Options) error{InvalidOptions}!void { - comptime std.debug.assert(backend.max_read_request_size * max_block_parallelism <= host_memory.Workspace.mapped_bytes_ceiling); - if (opts.block_parallelism == 0 or opts.block_parallelism > max_block_parallelism) { - log.err("invalid DMA calibration options: block_parallelism={d}, expected 1..{d}: InvalidOptions", .{ opts.block_parallelism, max_block_parallelism }); - return error.InvalidOptions; - } - if (opts.block_sizes.len == 0) { - log.err("invalid DMA calibration options: block_sizes is empty: InvalidOptions", .{}); - return error.InvalidOptions; - } - for (opts.block_sizes) |block_size| { - if (block_size == 0 or block_size > backend.max_read_request_size) { - log.err("invalid DMA calibration options: block size {d}, expected 1..{d}: InvalidOptions", .{ block_size, backend.max_read_request_size }); - return error.InvalidOptions; - } - } - if (opts.minimum_duration.nanoseconds <= 0) { - log.err("invalid DMA calibration options: minimum_duration={f}, expected more than zero: InvalidOptions", .{opts.minimum_duration}); - return error.InvalidOptions; - } - // Written so that NaN fails too. - if (!(opts.block_selection_tolerance >= 0 and opts.block_selection_tolerance < 1)) { - log.err("invalid DMA calibration options: block_selection_tolerance={d}, expected [0, 1): InvalidOptions", .{opts.block_selection_tolerance}); - return error.InvalidOptions; - } -} - -/// Measures one representative device into the direct loader's workspace -/// and returns the block it selects; CPU returns the smallest without -/// measuring. On a platform mixing device kinds, device 0's block serves -/// every device: it is a tuning choice, valid for all of them. `opts` were -/// checked (`validate`, or `zml.io.Loader.init` for the loader's block -/// sizes). The rate the `dma_bench` line logs is that one device's -/// synthetic H2D rate, not the platform's nor a prediction of a -/// checkpoint's load speed. -pub fn calibrate( - workspace: *host_memory.Workspace, - platform: *const Platform, - opts: Options, -) !usize { - // Every candidate loads alike on CPU: four PJRT CPU devices loaded warm - // sharded Llama in 1.55-1.60 s at 2, 8 and 16 MiB, so the smallest, - // which maps the least, is taken unmeasured and the loader grows its - // own arenas. - if (platform.target == .cpu) return std.mem.min(usize, opts.block_sizes); - - const allocator = workspace.allocator; - const io = workspace.io; - const start: std.Io.Timestamp = .now(io, .awake); - - var max_total_bytes: usize = 0; - for (opts.block_sizes) |block_size| { - max_total_bytes = @max(max_total_bytes, block_size * opts.block_parallelism); - } - // Map one ring for the largest candidate at the fixed transfer width; - // every candidate reuses it, and the load keeps it. - const calibration_source = try workspace.allocate(max_total_bytes); - - var session = try Session.init(allocator, io, platform, opts.block_sizes, opts.block_parallelism); - defer session.deinit(); - - const candidates = try allocator.alloc(Candidate, opts.block_sizes.len); - defer allocator.free(candidates); - for (candidates, opts.block_sizes, 0..) |*candidate, block_size, index| { - candidate.* = .{ .block_size = block_size, .buffer_offset = index * opts.block_parallelism }; - } - try measureAllCandidates(&session, opts, candidates, calibration_source); - const winner = candidates[try selectCandidate(&session, opts, candidates, calibration_source)]; - - log.debug("dma_bench platform={s} devices={d} kind=\"{s}\" block_bytes={d} parallelism={d} measured_gib_s={d:.3} duration={f} retained_mapped_bytes={d}", .{ - @tagName(platform.target), - platform.devices.len, - platform.devices[0].kind(), - winner.block_size, - opts.block_parallelism, - winner.median().bytesPerSecond() / (1024 * 1024 * 1024), - start.durationTo(.now(io, .awake)), - workspace.mappedBytes(), - }); - return winner.block_size; -} - -const sample_count = 3; - -fn measureAllCandidates( - session: *Session, - opts: Options, - candidates: []Candidate, - source: []const u8, -) !void { - for (0..sample_count) |round| { - for (0..candidates.len) |i| { - const candidate = &candidates[(i + round) % candidates.len]; - candidate.metrics[round] = try runWindow( - session.io, - session, - candidate, - source, - opts.minimum_duration, - opts.minimum_transfers, - ); - } - } -} - -/// Returns the index of the selected candidate (`pick`), over each -/// candidate's median ratio to the peak, or its confirmation's when it is -/// borderline. -fn selectCandidate( - session: *Session, - opts: Options, - candidates: []Candidate, - source: []const u8, -) !usize { - const ratios = try session.allocator.alloc(f64, candidates.len); - defer session.allocator.free(ratios); - - var peak_index: usize = 0; - var peak_rate = candidates[0].median().bytesPerSecond(); - for (candidates[1..], 1..) |candidate, index| { - const rate = candidate.median().bytesPerSecond(); - if (rate > peak_rate) { - peak_index = index; - peak_rate = rate; - } - } - for (candidates, ratios) |candidate, *ratio| { - ratio.* = if (peak_rate == 0) 0 else candidate.median().bytesPerSecond() / peak_rate; - } - - // A borderline candidate's ratio is its confirmation's against the peak. - for (candidates, ratios, 0..) |*candidate, *ratio, index| { - if (index != peak_index and isBorderline(candidates, index, ratio.*, opts.block_selection_tolerance)) - ratio.* = try confirmCandidateAgainstPeak(session, candidate, &candidates[peak_index], source); - } - return pick(candidates, ratios, opts.block_selection_tolerance); -} - -/// The smallest block whose ratio is within `tolerance` of the best ratio, -/// screened or confirmed. A confirmation can rate a block above the screen's -/// peak, so the best ratio may belong to a larger block. -fn pick(candidates: []const Candidate, ratios: []const f64, tolerance: f64) usize { - var best_index: usize = 0; - for (ratios, 0..) |ratio, index| { - if (ratio > ratios[best_index]) best_index = index; - } - const floor = ratios[best_index] * (1.0 - tolerance); - var selected_index = best_index; - for (candidates, ratios, 0..) |candidate, ratio, index| { - if (ratio >= floor and candidate.block_size < candidates[selected_index].block_size) - selected_index = index; - } - return selected_index; -} - -/// Whether `candidates[index]`, at screen `ratio` to the peak, gets -/// confirmation windows (`confirmation_margin`). -fn isBorderline(candidates: []const Candidate, index: usize, ratio: f64, tolerance: f64) bool { - // Whether the candidate is near-peak in one round but fails in others. - var qualified_once = false; - var rejected_once = false; - for (candidates[index].metrics, 0..) |metric, round| { - var round_peak_rate: f64 = 0; - for (candidates) |round_candidate| { - round_peak_rate = @max(round_peak_rate, round_candidate.metrics[round].bytesPerSecond()); - } - const round_ratio = if (round_peak_rate == 0) 0 else metric.bytesPerSecond() / round_peak_rate; - if (round_ratio >= 1.0 - tolerance) - qualified_once = true - else - rejected_once = true; - } - return (qualified_once and rejected_once) or @abs(ratio - (1.0 - tolerance)) <= confirmation_margin; -} - -/// The median, over `sample_count` paired windows, of `candidate`'s rate -/// over `peak`'s. -fn confirmCandidateAgainstPeak( - session: *Session, - candidate: *Candidate, - peak: *Candidate, - source: []const u8, -) !f64 { - var ratios: [sample_count]f64 = undefined; - for (&ratios, 0..) |*ratio, repeat| { - const order = if (repeat % 2 == 0) - [_]*Candidate{ candidate, peak } - else - [_]*Candidate{ peak, candidate }; - var candidate_rate: f64 = undefined; - var peak_rate: f64 = undefined; - for (order) |measured| { - const rate = (try runWindow( - session.io, - session, - measured, - source, - confirmation_duration, - confirmation_minimum_transfers, - )).bytesPerSecond(); - if (measured == candidate) candidate_rate = rate else peak_rate = rate; - } - ratio.* = if (peak_rate == 0) 0 else candidate_rate / peak_rate; - } - std.mem.sort(f64, &ratios, {}, std.sort.asc(f64)); - return ratios[ratios.len / 2]; -} - -/// Runs one window of `candidate`'s transfers, `session.parallelism` at a -/// time. `io` sleeps and joins the window; the workers transfer on the -/// session's. -fn runWindow( - io: std.Io, - session: *Session, - candidate: *Candidate, - source: []const u8, - minimum_duration: std.Io.Duration, - minimum_transfers: u64, -) !Measurement { - const Worker = struct { - session: *Session, - buffer_index: usize, - block: []const u8, - metrics: *Counters, - ready: *std.atomic.Value(usize), - start: *std.Io.Event, - stop: *std.atomic.Value(bool), - - fn run(self: @This()) void { - _ = self.ready.fetchAdd(1, .release); - self.start.waitUncancelable(self.session.io); - while (!self.stop.load(.acquire)) { - self.session.transfer(self.buffer_index, self.block, self.metrics); - if (self.session.first_error.get() != null) return; - } - } - }; - - var metrics: Counters = .{}; - var ready: std.atomic.Value(usize) = .init(0); - var start: std.Io.Event = .unset; - var stop: std.atomic.Value(bool) = .init(false); - var group: std.Io.Group = .init; - defer { - // Workers borrow this window's stack. Every exit, including a canceled - // sleep or partial spawn, must release the start gate and join them. - stop.store(true, .release); - start.set(io); - group.await(io) catch {}; - } - for (0..session.parallelism) |slot| { - try group.concurrent(io, Worker.run, .{Worker{ - .session = session, - .buffer_index = candidate.buffer_offset + slot, - .block = candidate.block(source, slot), - .metrics = &metrics, - .ready = &ready, - .start = &start, - .stop = &stop, - }}); - } - - try session.warmup(candidate, source); - - while (ready.load(.acquire) != session.parallelism) try io.sleep(.fromMicroseconds(100), .awake); - const start_at: std.Io.Timestamp = .now(io, .awake); - start.set(io); - while (true) { - const elapsed = start_at.durationTo(.now(io, .awake)); - if (session.first_error.get()) |err| return err; - // is window complete - if (elapsed.nanoseconds >= minimum_duration.nanoseconds and metrics.transfers.load(.acquire) >= minimum_transfers) break; - try io.sleep(.fromMilliseconds(1), .awake); - } - stop.store(true, .release); - try group.await(io); - const elapsed = start_at.durationTo(.now(io, .awake)); - if (session.first_error.get()) |err| return err; - return .{ .bytes = metrics.bytes.load(.acquire), .elapsed = elapsed }; -} - -/// One block size and its screen measurements. Its slots are the session's -/// buffers from `buffer_offset` on, one per transfer in flight. -const Candidate = struct { - block_size: usize, - buffer_offset: usize, - /// Slots warmed so far (`Session.warmup`). - warmed_buffers: usize = 0, - - /// One window per screen round (`measureAllCandidates`). - metrics: [sample_count]Measurement = undefined, - - fn median(self: Candidate) Measurement { - var sorted = self.metrics; - std.mem.sort(Measurement, &sorted, {}, struct { - fn lessThan(_: void, lhs: Measurement, rhs: Measurement) bool { - return lhs.bytesPerSecond() < rhs.bytesPerSecond(); - } - }.lessThan); - return sorted[sorted.len / 2]; - } - - /// The ring's bytes that `slot` transfers. - fn block(self: Candidate, source: []const u8, slot: usize) []const u8 { - return source[slot * self.block_size ..][0..self.block_size]; - } -}; - -/// One transfer manager for the whole screen, sliced per candidate, and the -/// screen's first transfer error, which ends it. -const Session = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - manager: *pjrt.AsyncHostToDeviceTransferManager, - buffers: []*pjrt.Buffer, - /// Each candidate's slots, and the transfers a window runs at a time. - parallelism: usize, - first_error: stdx.FirstError(pjrt.ApiError) = .{}, - - fn transfer(self: *Session, buffer_index: usize, block: []const u8, metrics: ?*Counters) void { - const api = self.platform.pjrt_api; - const event = self.manager.transferData(api, buffer_index, block, 0, false) catch |err| { - _ = self.first_error.set(err); - return; - }; - defer event.deinit(api); - event.await(api, self.io) catch |err| { - _ = self.first_error.set(err); - return; - }; - if (metrics) |output| { - _ = output.bytes.fetchAdd(@intCast(block.len), .monotonic); - _ = output.transfers.fetchAdd(1, .monotonic); - } - } - - /// Transfers twice into each of `candidate`'s slots not yet warmed. - fn warmup(self: *Session, candidate: *Candidate, source: []const u8) !void { - while (candidate.warmed_buffers < self.parallelism) : (candidate.warmed_buffers += 1) { - const slot = candidate.warmed_buffers; - self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); - self.transfer(candidate.buffer_offset + slot, candidate.block(source, slot), null); - if (self.first_error.get()) |err| return err; - } - } - - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - block_sizes: []const usize, - parallelism: usize, - ) !Session { - const buffer_count = std.math.mul(usize, block_sizes.len, parallelism) catch return error.OutOfMemory; - const dims = try allocator.alloc([1]i64, block_sizes.len); - defer allocator.free(dims); - const specs = try allocator.alloc(pjrt.ShapeSpec, buffer_count); - defer allocator.free(specs); - for (block_sizes, dims, 0..) |block_size, *dim, index| { - dim.* = .{@intCast(block_size)}; - @memset(specs[index * parallelism ..][0..parallelism], pjrt.ShapeSpec.init(dim, .u8)); - } - const buffers = try allocator.alloc(*pjrt.Buffer, buffer_count); - errdefer allocator.free(buffers); - const manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ - .shape_specs = specs, - .memory = platform.devices[0].memory(.default).?.pjrt_memory, - }); - var retrieved: usize = 0; - errdefer { - manager.deinit(platform.pjrt_api); - for (buffers[0..retrieved]) |buffer| buffer.deinit(platform.pjrt_api); - } - for (buffers, 0..) |*buffer, index| { - buffer.* = try manager.retrieveBuffer(platform.pjrt_api, index); - retrieved += 1; - } - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .manager = manager, - .buffers = buffers, - .parallelism = parallelism, - }; - } - - fn deinit(self: *Session) void { - // XLA commit 1b19ae012aa67426658f7ca3c1503bb781c863a9 switched GPU - // transfers to CommonAsyncHostToDeviceTransferManager, whose destructor - // waits for outstanding transfers and marks unfinished buffers as errored. - // We discard these buffers, so no final is_last_transfer=true copy is needed. - self.manager.deinit(self.platform.pjrt_api); - for (self.buffers) |buffer| buffer.deinit(self.platform.pjrt_api); - self.allocator.free(self.buffers); - self.* = undefined; - } -}; - -const Counters = struct { - bytes: std.atomic.Value(u64) = .init(0), - transfers: std.atomic.Value(u64) = .init(0), -}; - -const Measurement = struct { - bytes: u64, - elapsed: std.Io.Duration, - - fn bytesPerSecond(self: Measurement) f64 { - if (self.elapsed.nanoseconds <= 0) return 0; - return @as(f64, @floatFromInt(self.bytes)) * std.time.ns_per_s / - @as(f64, @floatFromInt(self.elapsed.nanoseconds)); - } -}; - -test "DMA benchmark selection uses medians and prefers the smallest near-peak value" { - // No candidate below is borderline, so the selector never schedules a - // confirmation window and the session supplies only its allocator. - var session: Session = .{ - .allocator = std.testing.allocator, - .io = undefined, - .platform = undefined, - .manager = undefined, - .buffers = undefined, - .parallelism = undefined, - }; - const opts: Options = .{ .block_selection_tolerance = 0.05 }; - - var candidates = [_]Candidate{ - .{ .block_size = 2, .buffer_offset = undefined }, - .{ .block_size = 4, .buffer_offset = undefined }, - .{ .block_size = 8, .buffer_offset = undefined }, - }; - const rates = [_][sample_count]u64{ - .{ 60, 10, 62 }, - .{ 98, 99, 97 }, - .{ 100, 101, 99 }, - }; - for (&candidates, rates) |*candidate, candidate_rates| { - for (&candidate.metrics, candidate_rates) |*metric, rate| { - metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; - } - } - const winner = candidates[try selectCandidate(&session, opts, &candidates, &.{})]; - try std.testing.expectEqual(@as(usize, 4), winner.block_size); - try std.testing.expectEqual(@as(f64, 98), winner.median().bytesPerSecond()); - - // A larger block near the peak in every round is neither confirmed nor - // picked. - var bimodal = [_]Candidate{ - .{ .block_size = 2, .buffer_offset = undefined }, - .{ .block_size = 4, .buffer_offset = undefined }, - .{ .block_size = 8, .buffer_offset = undefined }, - .{ .block_size = 16, .buffer_offset = undefined }, - }; - const bimodal_rates = [_]u64{ 80, 100, 70, 99 }; - for (&bimodal, bimodal_rates) |*candidate, rate| { - candidate.metrics = @splat(.{ .bytes = rate, .elapsed = .fromSeconds(1) }); - } - try std.testing.expectEqual(@as(usize, 4), bimodal[try selectCandidate(&session, opts, &bimodal, &.{})].block_size); -} - -test "DMA benchmark picks the smallest block within the tolerance of the best ratio" { - const candidates = [_]Candidate{ - .{ .block_size = 16 * 1024 * 1024, .buffer_offset = undefined }, - .{ .block_size = 32 * 1024 * 1024, .buffer_offset = undefined }, - }; - // 32 MiB is the peak; 16 MiB within the tolerance below it, then past it. - try std.testing.expectEqual(0, pick(&candidates, &.{ 0.93, 1.0 }, 0.08)); - try std.testing.expectEqual(1, pick(&candidates, &.{ 0.91, 1.0 }, 0.08)); - // 16 MiB is the screen's peak; a confirmation rates 32 MiB more than the - // tolerance above it, then within it. - try std.testing.expectEqual(1, pick(&candidates, &.{ 1.0, 1.10 }, 0.08)); - try std.testing.expectEqual(0, pick(&candidates, &.{ 1.0, 1.05 }, 0.08)); -} - -test "DMA benchmark confirms a candidate when round qualification disagrees" { - var candidates = [_]Candidate{ - .{ .block_size = 4, .buffer_offset = undefined }, - .{ .block_size = 8, .buffer_offset = undefined }, - }; - const rates = [_][sample_count]u64{ - .{ 99, 80, 70 }, - .{ 100, 100, 100 }, - }; - for (&candidates, rates) |*candidate, candidate_rates| { - for (&candidate.metrics, candidate_rates) |*metric, rate| { - metric.* = .{ .bytes = rate, .elapsed = .fromSeconds(1) }; - } - } - // The median ratio, 0.80, is far from the 0.95 floor: only the rounds - // make it borderline. - try std.testing.expect(isBorderline(&candidates, 0, 0.80, 0.05)); -} - -test "measure refuses a knob out of range before it touches the platform" { - const cases = [_]Options{ - .{ .block_sizes = &.{} }, - .{ .block_sizes = &.{0} }, - .{ .block_sizes = &.{backend.max_read_request_size + 1} }, - .{ .block_parallelism = 0 }, - .{ .block_parallelism = max_block_parallelism + 1 }, - .{ .minimum_duration = .zero }, - .{ .minimum_duration = .fromMilliseconds(-1) }, - .{ .block_selection_tolerance = 1 }, - .{ .block_selection_tolerance = -0.01 }, - .{ .block_selection_tolerance = std.math.nan(f64) }, - }; - for (cases) |opts| { - try std.testing.expectError(error.InvalidOptions, measure(std.testing.allocator, std.testing.io, undefined, opts)); - } -} - -test "measure needs no loader and selects the smallest block on CPU" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - try std.testing.expectEqual(4 << 20, try measure(allocator, io, platform, .{ .block_sizes = &.{ 8 << 20, 4 << 20 } })); -} - -test "DMA benchmark shares one manager across candidate sizes and slots" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - - var session = try Session.init(allocator, io, platform, &.{ 8, 16 }, 2); - defer session.deinit(); - const source: [32]u8 = @splat(0); - for ([_]usize{ 8, 16 }, 0..) |block_size, index| { - var candidate: Candidate = .{ .block_size = block_size, .buffer_offset = index * 2 }; - try session.warmup(&candidate, &source); - try std.testing.expectEqual(2, candidate.warmed_buffers); - } - try std.testing.expectEqual(null, session.first_error.get()); -} - -test "DMA benchmark cancellation drains transfer workers" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = Platform.auto(allocator, io, .{ .cpu = .{ .device_count = 1 } }) catch - return error.SkipZigTest; - defer platform.deinit(allocator, io); - - var source: [16]u8 = @splat(0); - var session = try Session.init(allocator, io, platform, &.{source.len}, 1); - defer session.deinit(); - var candidate: Candidate = .{ .block_size = source.len, .buffer_offset = 0 }; - var vtable = io.vtable.*; - vtable.sleep = struct { - fn sleep(_: ?*anyopaque, _: std.Io.Timeout) std.Io.Cancelable!void { - return error.Canceled; - } - }.sleep; - const canceled_io: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; - - // A cancellation can arrive while workers wait for the start gate or - // while they transfer. In either case they must finish before teardown. - try std.testing.expectError(error.Canceled, runWindow( - canceled_io, - &session, - &candidate, - &source, - .max, - 0, - )); -} diff --git a/zml/io/host_memory.zig b/zml/io/host_memory.zig deleted file mode 100644 index 45063b7e..00000000 --- a/zml/io/host_memory.zig +++ /dev/null @@ -1,613 +0,0 @@ -//! Loader-owned host arenas, cut into blocks once. The direct backend owns -//! its workspace for its whole life: calibration maps its ring there, the -//! backend grows the set to what its load holds and carves it, and its -//! `destroy` releases every arena. Callers never supply a workspace, a -//! budget, a NUMA policy or a calibration result. -//! -//! The memory depends on the target: CUDA and oneAPI take page-backed arenas -//! registered with PJRT (DmaMap), ROCm and Furiosa PJRT `pinned_host` -//! buffers, and CPU plain pages, unregistered. The arenas' kind and the -//! transfer path are separate decisions: CPU loads through the direct -//! backend too. When the task's cpuset and memory policy allow several -//! memory-bearing NUMA nodes, page-backed arenas interleave over those of the -//! platform's devices (all of them when the load reads past the page cache, -//! a device's node is unknown, or none of theirs is allowed); ROCm balances -//! whole arenas across its devices' host nodes. -//! `Workspace.mapped_bytes_ceiling` (16 GiB) is a safety guard, not a -//! target. A failed allocation or registration, or a growth past the -//! ceiling, is logged with its cause and returns `OutOfMemory`. - -const std = @import("std"); -const Alignment = std.mem.Alignment; -const Allocator = std.mem.Allocator; -const builtin = @import("builtin"); - -const pjrt = @import("pjrt"); - -const Platform = @import("../platform.zig").Platform; -const log = @import("log.zig").mem; - -/// One ROCm host-memory allocation path and its bytes allocated so far. -const HostNode = struct { - /// The plugin's `numa_node` device attribute (`Device.numaNode`); null - /// for the one path kept when no device reports it. - node: ?u32, - any_device_index: usize, - total_allocated_bytes: usize = 0, -}; - -/// One retained arena. The bytes of a `pinned_host` arena belong to its -/// PJRT buffer; page-backed arenas have none. -const Arena = struct { - bytes: []u8, - pjrt_buffer: ?*pjrt.Buffer = null, -}; - -/// How arenas are mapped and unmapped; each strategy keeps only the state -/// it uses. -const ArenaKind = union(enum) { - /// With eight MI300X visible, hipHostRegister took ~6.3 s for 1 GiB - /// versus 0.6-0.7 s through hipHostMalloc. KFD/IOMMU registration maps - /// pages to every GPU; huge-page advice helped only ~9%, and touching - /// pages or changing HIP flags did not remove the cost. Standard PJRT - /// pinned_host buffers provide allocation-owned memory without a custom - /// allocator extension. - pjrt_host: struct { - platform: *const Platform, - host_nodes: std.ArrayList(HostNode), - }, - /// Page-backed arenas, registered with the platform's PJRT client when - /// there is one (`dma_map` in the arena log) and left pageable for CPU - /// transfers when not (`pageable`), with huge-page alignment and advice. - /// Registration only avoids staging if the plugin recognizes the range as - /// pinned, including subranges. The oneAPI plugin must implement - /// IsHostMemoryPinned for SYCL imports: without it, one B70 staged - /// transfers at 11.01 GiB/s instead of 26.90 (eight DMA caps). - pages: struct { - platform: ?*const Platform, - /// Zero leaves placement to the kernel. Automatic placement can - /// fall back to zero if the kernel refuses it. - numa_mask: u64, - }, - - const transparent_huge_page_size = 2 * 1024 * 1024; - const mpol_interleave = 3; - - fn init( - allocator: Allocator, - io: std.Io, - platform: *const Platform, - direct_reads: bool, - ) Allocator.Error!ArenaKind { - return switch (platform.target) { - .cuda, .oneapi => .{ .pages = .{ .platform = platform, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, - .cpu => .{ .pages = .{ .platform = null, .numa_mask = pagesNumaMask(allocator, io, platform, direct_reads) } }, - // Furiosa backs `.host_pinned` with SDK pinned (dma-buf) memory - // from `/dev/dma_heap/system`, which the SDK recognizes by - // address and transfers without pinning pages on every request. - .rocm, .furiosa => { - // PJRT-pinned allocations choose placement through the device's - // host memory space, not our mbind policy. Retain one allocation - // path per reported node; the blocks stay unpartitioned. - var host_nodes: std.ArrayList(HostNode) = try .initCapacity(allocator, @max(platform.devices.len, 1)); - devices: for (platform.devices, 0..) |device, device_index| { - std.debug.assert(device.memory(.host_pinned) != null); - const node = device.numaNode() orelse continue; - for (host_nodes.items) |existing| { - if (existing.node == node) continue :devices; - } - host_nodes.appendAssumeCapacity(.{ .node = node, .any_device_index = device_index }); - } - if (host_nodes.items.len == 0) host_nodes.appendAssumeCapacity(.{ .node = null, .any_device_index = 0 }); - return .{ .pjrt_host = .{ .platform = platform, .host_nodes = host_nodes } }; - }, - .tpu, .neuron, .metal => @panic("DMA is not accessible on this platform and shouldn't have been called."), - }; - } - - /// Call after every arena is unmapped. - fn deinit(self: *ArenaKind, allocator: Allocator) void { - switch (self.*) { - .pjrt_host => |*host| host.host_nodes.deinit(allocator), - .pages => {}, - } - } - - /// Maps one arena of `bytes`. - fn map(self: *ArenaKind, allocator: Allocator, io: std.Io, bytes: usize) Allocator.Error!Arena { - const started: std.Io.Timestamp = .now(io, .awake); - switch (self.*) { - .pjrt_host => |*host| { - // Balance bytes, not arena/device counts: arenas differ in size. - // Replicated Llama-3.1-8B on eight MI300X lost the fast mode when - // all arenas came from device zero (1.27-1.45 s). Device rotation - // left a 61/39 byte split and took 0.98-1.33 s; byte-balanced - // allocation recovered 0.84-1.05 s over six runs, with 1.52 GiB - // mapped versus 2.03 GiB for per-node pools. Host contention - // affected both arms. - var host_node = &host.host_nodes.items[0]; - for (host.host_nodes.items[1..]) |*candidate| { - if (candidate.total_allocated_bytes < host_node.total_allocated_bytes) host_node = candidate; - } - const device_index = host_node.any_device_index; - // PJRT is the sole owner of these hipHostMalloc-backed bytes. - // Never DmaMap or DmaUnmap the borrowed pointer: unregistering - // allocation-owned memory breaks the eventual hipHostFree. The - // buffer and its external reference both live until `unmap`, - // after all reads and DMA finish. - // The ROCm plugin must implement IsHostMemoryPinned: without - // it, one MI300X staged transfers at 6.5 GiB/s instead of 46.6. - const api = host.platform.pjrt_api; - const memory = host.platform.devices[device_index].memory(.host_pinned).?; - const buffer = host.platform.pjrt_client.createUninitializedBuffer(api, .{ - .dims = &.{@intCast(bytes)}, - .element_type = .u8, - .layout = .{ - .tiled = .{ - .minor_to_major = &.{0}, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .dst = .{ .memory = memory.pjrt_memory }, - }) catch |err| { - log.err("pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); - return error.OutOfMemory; - }; - errdefer buffer.deinit(api); - std.debug.assert(buffer.isOnCpu(api)); - buffer.increaseExternalReferenceCount(api) catch |err| { - log.err("retaining pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); - return error.OutOfMemory; - }; - errdefer buffer.decreaseExternalReferenceCount(api) catch {}; - const ptr: [*]u8 = @ptrCast(buffer.opaqueDeviceMemoryDataPointer(api) catch |err| { - log.err("accessing pinned host allocation failed on device {d} for {Bi:.2}: {s}", .{ device_index, bytes, @errorName(err) }); - return error.OutOfMemory; - }); - // Counted once mapped: the workspace retains every mapped arena. - host_node.total_allocated_bytes += bytes; - log.debug("DMA arena kind=pjrt_host device={d} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ - device_index, - @intFromPtr(ptr), - bytes, - @as(f64, @floatFromInt(elapsedNanoseconds(started, .now(io, .awake)))) / std.time.ns_per_ms, - }); - return .{ .bytes = ptr[0..bytes], .pjrt_buffer = buffer }; - }, - .pages => |*pages| { - const alignment = pageAlignment(bytes); - const ptr = allocator.rawAlloc(bytes, alignment, @returnAddress()) orelse { - log.err("host memory allocation failed for {Bi:.2}: OutOfMemory", .{bytes}); - return error.OutOfMemory; - }; - const data = ptr[0..bytes]; - place(&pages.numa_mask, data); - adviseHugePages(data); - if (pages.platform) |p| p.pjrt_client.dmaMap(p.pjrt_api, data) catch |err| { - log.err("DMA registration failed for {Bi:.2} of host memory: {s}", .{ bytes, @errorName(err) }); - allocator.rawFree(data, alignment, @returnAddress()); - return error.OutOfMemory; - }; - const finished: std.Io.Timestamp = .now(io, .awake); - log.debug("DMA arena kind={s} placement={s} nodes=0x{x} address=0x{x} size={Bi:.2} allocation_ms={d:.3}", .{ - if (pages.platform != null) "dma_map" else "pageable", - if (pages.numa_mask == 0) "unplaced" else "interleave", - pages.numa_mask, - @intFromPtr(data.ptr), - bytes, - @as(f64, @floatFromInt(elapsedNanoseconds(started, finished))) / std.time.ns_per_ms, - }); - return .{ .bytes = data }; - }, - } - } - - fn unmap(self: *const ArenaKind, allocator: Allocator, arena: Arena) void { - switch (self.*) { - .pjrt_host => |host| { - const api = host.platform.pjrt_api; - const buffer = arena.pjrt_buffer.?; - buffer.decreaseExternalReferenceCount(api) catch unreachable; - buffer.deinit(api); - }, - .pages => |pages| { - if (pages.platform) |platform| { - // Unregistering can fail after a device error. The pages may - // still be registered, so they are leaked, not reused. - platform.pjrt_client.dmaUnmap(platform.pjrt_api, arena.bytes) catch |err| { - log.warn("DMA unregistration failed for {Bi:.2} of host memory, which stays allocated: {s}", .{ arena.bytes.len, @errorName(err) }); - return; - }; - } - allocator.rawFree(arena.bytes, pageAlignment(arena.bytes.len), @returnAddress()); - }, - } - } - - /// Page alignment, raised to the huge-page size for arenas that span one. - fn pageAlignment(len: usize) Alignment { - const alignment: Alignment = comptime .fromByteUnits(std.heap.page_size_min); - if (comptime builtin.os.tag != .linux) return alignment; - if (len < transparent_huge_page_size) return alignment; - return alignment.max(.fromByteUnits(transparent_huge_page_size)); - } - - /// Interleaves `data` over `numa_mask`'s nodes; on refusal, warns and - /// zeroes the mask, so later arenas stay unplaced too. - fn place(numa_mask: *u64, data: []u8) void { - const mask = numa_mask.*; - if (mask == 0) return; - if (comptime builtin.os.tag != .linux) return; - - const node_mask: [1]u64 = .{mask}; - const highest_node: usize = 63 - @clz(mask); - const rc = std.os.linux.syscall6( - .mbind, - @intFromPtr(data.ptr), - data.len, - mpol_interleave, - @intFromPtr(&node_mask), - // Linux get_nodes() decrements maxnode before copying the mask; - // raw callers include the same extra sentinel bit as libnuma. - highest_node + 2, - 0, - ); - if (std.os.linux.errno(rc) == .SUCCESS) return; - log.warn("NUMA placement of DMA arenas over nodes 0x{x} refused ({s}); leaving them unplaced", .{ - mask, - @tagName(std.os.linux.errno(rc)), - }); - // Automatic placement is not retried: the arenas stay unplaced. - numa_mask.* = 0; - } - - fn adviseHugePages(data: []u8) void { - if (comptime builtin.os.tag != .linux) return; - if (data.len < transparent_huge_page_size) return; - - const ptr: [*]align(std.heap.page_size_min) u8 = @alignCast(data.ptr); - std.posix.madvise(ptr, data.len, std.posix.MADV.HUGEPAGE) catch |err| { - log.warn("MADV_HUGEPAGE failed for DMA buffer at 0x{x} ({Bi:.2}): {s}", .{ - @intFromPtr(data.ptr), - data.len, - @errorName(err), - }); - }; - } -}; - -/// Host arenas owned by one direct loader. Calibration maps its ring -/// (`allocate`); the loader then grows the set to what its load holds and -/// cuts it into blocks, once (`carve`). Deinitialize after all transfers -/// finish and before the platform. -pub const Workspace = struct { - /// Safety guard on the arenas' total host memory (pinned on the DMA - /// targets), not an allocation target: calibration and the load's - /// pre-growth decide what is actually mapped. Fixed, because no caller - /// ever had a reason to choose another value. - pub const mapped_bytes_ceiling: usize = 16 * 1024 * 1024 * 1024; - - allocator: Allocator, - io: std.Io, - kind: ArenaKind, - /// Every arena mapped, in mapping order, until `deinit`. - arenas: std.ArrayList(Arena) = .empty, - /// The arenas cut into blocks, set by `carve`: nothing maps after it. - blocks: ?[]Block = null, - - /// `direct_reads`: the load reads its checkpoint past the page cache, so - /// page-backed arenas spread over every candidate node - /// (`placementMask`). - pub fn init( - allocator: Allocator, - io: std.Io, - platform: *const Platform, - direct_reads: bool, - ) Allocator.Error!Workspace { - return .{ - .allocator = allocator, - .io = io, - .kind = try .init(allocator, io, platform, direct_reads), - }; - } - - pub fn deinit(self: *Workspace) void { - const io = self.io; - const mapped_bytes = self.mappedBytes(); - const started: std.Io.Timestamp = .now(io, .awake); - if (self.blocks) |blocks| self.allocator.free(blocks); - for (self.arenas.items) |arena| self.kind.unmap(self.allocator, arena); - self.arenas.deinit(self.allocator); - self.kind.deinit(self.allocator); - const elapsed_ns = elapsedNanoseconds(started, .now(io, .awake)); - log.debug("DMA load workspace teardown: mapped={Bi:.2}, elapsed_ms={d:.3}", .{ - mapped_bytes, - @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms, - }); - self.* = undefined; - } - - /// The bytes of every retained arena, tails included. - pub fn mappedBytes(self: *const Workspace) usize { - var mapped: usize = 0; - for (self.arenas.items) |arena| mapped += arena.bytes.len; - return mapped; - } - - /// Retains one new arena within the mapped-byte ceiling, before `carve`. - pub fn allocate(self: *Workspace, bytes: usize) Allocator.Error![]u8 { - std.debug.assert(self.blocks == null); - const mapped = self.mappedBytes(); - if (bytes > mapped_bytes_ceiling - mapped) { - log.err("host memory allocation exceeds mapped ceiling: requested={Bi:.2}, mapped={Bi:.2}, ceiling={Bi:.2}: OutOfMemory", .{ bytes, mapped, mapped_bytes_ceiling }); - return error.OutOfMemory; - } - // Reserved first, so a mapped arena is always retained: the ROCm - // balancer counts the bytes it maps. - try self.arenas.ensureUnusedCapacity(self.allocator, 1); - const arena = try self.kind.map(self.allocator, self.io, bytes); - self.arenas.appendAssumeCapacity(arena); - return arena.bytes; - } - - /// The blocks of `block_size` the workspace can hold: those of its - /// retained arenas, and those the mapped ceiling still allows. - pub fn capacity(self: *const Workspace, block_size: usize) usize { - return self.usableBlocks(block_size) + (mapped_bytes_ceiling - self.mappedBytes()) / block_size; - } - - /// Maps the blocks missing below `first_blocks` as one arena, then those - /// missing below `total_blocks` as another, and cuts every retained arena - /// into blocks of `block_size`, once. The set is then complete: nothing - /// maps inside a load (a slab mapped there cost 146 to 230 ms of - /// hipHostMalloc on MI300X), and `allocate` asserts it. Arena tails - /// smaller than one block stay mapped and unused. Blocks are not split - /// per NUMA node: strict per-node pools duplicated source reserves and - /// constrained reads to the smallest node; placement belongs to arena - /// allocation. The workspace owns the list and the memory. - pub fn carve(self: *Workspace, block_size: usize, first_blocks: usize, total_blocks: usize) Allocator.Error![]const Block { - std.debug.assert(self.blocks == null and block_size != 0); - for ([_]usize{ first_blocks, total_blocks }) |target_blocks| { - const missing_blocks = target_blocks -| self.usableBlocks(block_size); - if (missing_blocks != 0) _ = try self.allocate(missing_blocks * block_size); - } - const blocks = try self.allocator.alloc(Block, self.usableBlocks(block_size)); - var next: usize = 0; - for (self.arenas.items) |arena| { - for (0..arena.bytes.len / block_size) |j| { - blocks[next] = arena.bytes[j * block_size ..][0..block_size]; - next += 1; - } - } - std.debug.assert(next == blocks.len); - self.blocks = blocks; - return blocks; - } - - /// Counts complete blocks across retained arenas; block_size must be nonzero. - fn usableBlocks(self: *const Workspace, block_size: usize) usize { - var usable: usize = 0; - for (self.arenas.items) |arena| usable += arena.bytes.len / block_size; - return usable; - } - - /// Creates ordinary allocator-backed arenas for tests without a PJRT platform. - fn initForTesting(allocator: Allocator, io: std.Io) Workspace { - return .{ - .allocator = allocator, - .io = io, - .kind = .{ .pages = .{ .platform = null, .numa_mask = 0 } }, - }; - } -}; - -pub const Block = []u8; - -/// Bits of `/sys/devices/system/node/has_memory`, or zero when unreadable. -/// Nodes 64 and above cannot be represented and are dropped. -/// Device-coherent HBM nodes are not candidates for host pages: the measured -/// four-GB300 topology had 34 NUMA nodes but only two with host memory. -fn memoryNodeMask(allocator: Allocator, io: std.Io) u64 { - if (comptime builtin.os.tag != .linux) return 0; - const contents = std.Io.Dir.cwd().readFileAlloc( - io, - "/sys/devices/system/node/has_memory", - allocator, - .limited(4096), - ) catch return 0; - defer allocator.free(contents); - return parseNodeList(contents); -} - -/// The nodes the task's memory policy allows (`numactl --membind`, -/// `--interleave`, `--preferred`); every node under the default policy. -/// An arena placed outside them would defeat the policy: `mbind` on a -/// range overrides `set_mempolicy`. -fn taskMemoryPolicyMask() u64 { - if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); - var mode: c_int = 0; - var nodes: [1]u64 = .{0}; - const rc = std.os.linux.syscall5(.get_mempolicy, @intFromPtr(&mode), @intFromPtr(&nodes), 64, 0, 0); - if (std.os.linux.errno(rc) != .SUCCESS or mode == 0 or nodes[0] == 0) return std.math.maxInt(u64); - return nodes[0]; -} - -/// The nodes the task's cpuset lets it allocate on (`Mems_allowed`), or -/// every node when unknown. `mbind` refuses a mask with none of them. -fn cpusetMemsMask() u64 { - if (comptime builtin.os.tag != .linux) return std.math.maxInt(u64); - const mpol_f_mems_allowed = 4; - var nodes: [1]u64 = .{0}; - const rc = std.os.linux.syscall5(.get_mempolicy, 0, @intFromPtr(&nodes), 64, 0, mpol_f_mems_allowed); - if (std.os.linux.errno(rc) != .SUCCESS or nodes[0] == 0) return std.math.maxInt(u64); - return nodes[0]; -} - -/// The nodes page-backed arenas interleave over (`placementMask`): the -/// memory-bearing nodes the task's cpuset and memory policy allow, narrowed -/// to the nodes of the platform's devices, unless the load reads past the -/// page cache. -fn pagesNumaMask(allocator: Allocator, io: std.Io, platform: *const Platform, direct_reads: bool) u64 { - const candidates = memoryNodeMask(allocator, io) & cpusetMemsMask() & taskMemoryPolicyMask(); - return placementMask(candidates, deviceNodeMask(platform), direct_reads); -} - -/// The nodes of the platform's devices (`Device.numaNode`), or null when a -/// device reports none (the CPU plugin never does) or a node past 63. -fn deviceNodeMask(platform: *const Platform) ?u64 { - var mask: u64 = 0; - for (platform.devices) |device| { - const node = device.numaNode() orelse return null; - if (node >= 64) return null; - mask |= @as(u64, 1) << @intCast(node); - } - return if (mask == 0) null else mask; -} - -/// Zero leaves placement to the kernel: with one candidate node or none -/// there is nothing to choose, and the task's policy or the only node decides. -/// For reads through the page cache, the arenas go to the devices' nodes -/// among several candidates: an arena on another node crosses the -/// inter-socket link on every H2D copy. Two GB300 on one Grace node loaded -/// Qwen3.8-27B-FP8 in 1.265 s with a task policy holding the arenas to that -/// node, against 1.64 s interleaved over both nodes. Devices on several -/// nodes share one pool interleaved over theirs: four GB300 on two nodes -/// loaded replicated DeepSeek-V4-Flash in 4.35 s against 4.92 s with -/// per-node pools. When a device's node is unknown, or none is a candidate, -/// the arenas interleave over every candidate, which no single node beat -/// everywhere. A single node is placed too, since unplaced CUDA arenas -/// follow the registering thread's node; the kernel still falls back when -/// it is full. -/// A load read past the page cache (`direct_reads`) spreads the arenas over -/// every candidate instead: the drives write into them, and interleaving -/// splits that traffic, and the copies out of them, between the candidates' -/// memory. Over an md RAID0 of four NVMe drives, two on each node, the same -/// two GB300 loaded the same checkpoint forced direct in 1.752 s with the -/// arenas interleaved over both nodes, against 1.873 s with them on the -/// devices' node: staging a read took 0.52 against 0.64 ms, and a 16 MiB -/// read 1.90 against 1.87 ms. The drives' own nodes are not consulted: every -/// candidate matched them on the one host measured. -fn placementMask(candidates: u64, device_nodes: ?u64, direct_reads: bool) u64 { - if (@popCount(candidates) < 2) return 0; - if (direct_reads) return candidates; - const near = candidates & (device_nodes orelse candidates); - return if (near != 0) near else candidates; -} - -/// Parses a kernel node list ("0-1", "0,2-3") into a mask; zero on any error. -fn parseNodeList(text: []const u8) u64 { - var mask: u64 = 0; - var ranges = std.mem.tokenizeAny(u8, text, ", \n"); - while (ranges.next()) |range| { - var ends = std.mem.splitScalar(u8, range, '-'); - const first = std.fmt.parseInt(usize, ends.first(), 10) catch return 0; - const last = if (ends.next()) |end| - std.fmt.parseInt(usize, end, 10) catch return 0 - else - first; - if (last < first or ends.next() != null) return 0; - var node = first; - while (node <= last) : (node += 1) { - if (node < 64) mask |= @as(u64, 1) << @intCast(node); - } - } - return mask; -} - -fn elapsedNanoseconds(started: std.Io.Timestamp, finished: std.Io.Timestamp) u64 { - return @intCast(@max(started.durationTo(finished).nanoseconds, 0)); -} - -test "automatic NUMA placement keeps the devices' nodes for buffered reads and spreads direct reads over every candidate" { - // One candidate or none: the kernel places. - try std.testing.expectEqual(@as(u64, 0), placementMask(0, 0b10, false)); - try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, false)); - try std.testing.expectEqual(@as(u64, 0), placementMask(0b1000, null, false)); - // Devices on one of two nodes: that node; on both: both. - try std.testing.expectEqual(@as(u64, 0b10), placementMask(0b11, 0b10, false)); - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, false)); - // Unknown device nodes, or none among the candidates: every candidate. - try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, false)); - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b100, false)); - // A cpuset of nodes 0 and 2 of four, devices on node 1: the cpuset's - // nodes, as `pagesNumaMask` narrows the candidates to them first. - try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, false)); - - // Direct reads: every candidate, wherever the devices are. - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b10, true)); - try std.testing.expectEqual(@as(u64, 0b11), placementMask(0b11, 0b11, true)); - try std.testing.expectEqual(@as(u64, 0b1001), placementMask(0b1001, null, true)); - // Still nothing to choose with one candidate, and still within the - // cpuset. - try std.testing.expectEqual(@as(u64, 0), placementMask(0b1, 0b1, true)); - try std.testing.expectEqual(@as(u64, 0b101), placementMask(0b1111 & 0b101, 0b10, true)); -} - -test "parseNodeList accepts kernel node lists" { - try std.testing.expectEqual(@as(u64, 0b11), parseNodeList("0-1\n")); - try std.testing.expectEqual(@as(u64, 0b1101), parseNodeList("0,2-3")); - try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0")); - try std.testing.expectEqual(@as(u64, 0), parseNodeList("")); - try std.testing.expectEqual(@as(u64, 0), parseNodeList("1-0")); - try std.testing.expectEqual(@as(u64, 0), parseNodeList("x")); - try std.testing.expectEqual(@as(u64, 0b1), parseNodeList("0,64")); -} - -test "Workspace capacity counts whole blocks under the mapped ceiling, and carve refuses one more" { - const allocator = std.testing.allocator; - var workspace: Workspace = .initForTesting(allocator, std.testing.io); - defer workspace.deinit(); - const ceiling = Workspace.mapped_bytes_ceiling; - - try std.testing.expectEqual(ceiling / 64, workspace.capacity(64)); - _ = try workspace.allocate(150); - _ = try workspace.allocate(127); - // Whole blocks only: the arenas hold three, their tails none. - try std.testing.expectEqual(3 + (ceiling - 277) / 64, workspace.capacity(64)); - // Capacity is tight, as direct `create` sizes its carve from it: one - // block more crosses the ceiling (reaching it would map 16 GiB). - try std.testing.expectError(error.OutOfMemory, workspace.allocate(ceiling - 277 + 1)); - try std.testing.expectError(error.OutOfMemory, workspace.carve(64, workspace.capacity(64) + 1, 0)); - try std.testing.expectEqual(@as(usize, 277), workspace.mappedBytes()); -} - -test "Workspace carve maps each step's missing blocks as one arena and leaves tails out" { - const allocator = std.testing.allocator; - var workspace: Workspace = .initForTesting(allocator, std.testing.io); - defer workspace.deinit(); - const first = try workspace.allocate(150); - const second = try workspace.allocate(127); - - // Three blocks are mapped: two more make the first five, three more the - // eight. - const blocks = try workspace.carve(64, 5, 8); - try std.testing.expectEqual(@as(usize, 8), blocks.len); - try std.testing.expectEqual(@as(usize, 4), workspace.arenas.items.len); - try std.testing.expectEqual(@as(usize, 2 * 64), workspace.arenas.items[2].bytes.len); - try std.testing.expectEqual(@as(usize, 3 * 64), workspace.arenas.items[3].bytes.len); - for (blocks) |block| try std.testing.expectEqual(@as(usize, 64), block.len); - // Tails stay mapped without contributing a block. - try std.testing.expectEqual(first.ptr, blocks[0].ptr); - try std.testing.expectEqual(first.ptr + 64, blocks[1].ptr); - try std.testing.expectEqual(second.ptr, blocks[2].ptr); -} - -test "Workspace arena ownership cleans up allocation failures" { - const AllocationTest = struct { - fn run(allocator: Allocator) !void { - var workspace: Workspace = .initForTesting(allocator, std.testing.io); - defer workspace.deinit(); - - _ = try workspace.allocate(64); - _ = try workspace.allocate(128); - // The first step is already mapped, as calibration's ring covers - // one device's reserve; the second maps the last block. - const blocks = try workspace.carve(64, 3, 4); - try std.testing.expectEqual(@as(usize, 4), blocks.len); - try std.testing.expectEqual(@as(usize, 256), workspace.mappedBytes()); - try std.testing.expectEqual(@as(usize, 3), workspace.arenas.items.len); - } - }; - try std.testing.checkAllAllocationFailures(std.testing.allocator, AllocationTest.run, .{}); -} diff --git a/zml/io/loader.zig b/zml/io/loader.zig deleted file mode 100644 index 7d63c4d8..00000000 --- a/zml/io/loader.zig +++ /dev/null @@ -1,2888 +0,0 @@ -//! Checkpoint loading. `Loader.start` takes a whole model: it starts -//! compiling the programs that compute some of its tensors -//! (`Source.bindProgram`) and publishes every tensor the loader fills -//! itself. `wait` then publishes the programs' inputs in groups the devices -//! have room for, runs each group's programs on the caller's task once its -//! inputs landed, and retires everything. Backends only read sources into -//! buffers; lookup, compilation, admission and execution stay in this -//! shared front end. The contract is in the pub doc comments below; -//! docs/learn/loader.md is the user guide. -//! -//! From bindings to specs. `start` checks every tensor and program input -//! once, before anything is published (`prepareSpec`, `checkPlacement`). -//! It lowers each bound source to bytes (`Source.Placed.lower`, the one -//! rule), one part per piece: a part of the destination's logical bytes -//! (the row-major bytes of its shape, before the sharding splits them), a -//! run of `width` bytes repeated over strided dimensions. A whole -//! checkpoint tensor is one part. -//! A strided part of at most `max_part_runs` runs, or of runs of at least -//! `large_run_bytes`, becomes one part per run; a spec with any other -//! strided part (row interleaves, transposes, broadcasts) goes to the -//! backend as strided parts, which the direct backend gathers. The parts go -//! in destination order, and the bytes none covers are zero. Each -//! `LoadSpec` is one destination, which the backends trust as -//! `backend.LoadSpec` states. Only `Source` makes bindings and its pieces -//! are disjoint by construction: nothing here checks that (Source's tests -//! do). -//! -//! Order and admission. The tensors the loader fills go out first, as one -//! submission, so every program group is admitted with them counted. A -//! group holds the tensors of consecutive programs in model order, up to -//! `program_group_bytes` of outputs, so the inputs of one layer's experts, -//! adjacent in the checkpoint, are read together. Before publishing a -//! group, `wait` waits for its programs to compile, then retires the oldest -//! pending submissions until the group fits the room every device reports -//! (`admission`): its limit, minus what is in use, minus the bytes the -//! direct backend published but has not allocated yet, minus a 64 MiB -//! reserve. A device's allocator may count freed memory as in use until it -//! is given back (`Platform.releaseFreedDeviceMemory`), so `start` gives -//! back what was freed before the load. A group costs its inputs, its -//! outputs and the largest of its programs' temporaries per device, since -//! they run one after the other. Once no program group is pending, retiring -//! frees nothing more: the room is read again after the retired groups' -//! memory is given back, and the group goes even if it is still short, -//! since the room is an estimate and can read low. A group that does not -//! fit a device fails its allocation or its execution with the device's -//! error, which `wait` returns; a smaller `Options.program_group_bytes` -//! makes smaller groups. Where a device reports no limit (CPU) or the -//! backend does not count its allocations (buffered), nothing is measured -//! and one program group is in flight at a time. -const std = @import("std"); - -const pjrt = @import("pjrt"); -const VFS = @import("vfs"); - -const Buffer = @import("../buffer.zig").Buffer; -const Compiler = @import("../Compiler.zig"); -const Exe = @import("../exe.zig").Exe; -const mem = @import("../mem.zig"); -const Bufferized = mem.Bufferized; -const meta = @import("../meta.zig"); -const ops = @import("../ops.zig"); -const platform_mod = @import("../platform.zig"); -const Memory = TensorStore.Memory; -const Platform = platform_mod.Platform; -const safetensors = @import("../safetensors.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; -const testing = @import("../testing.zig"); -const backend = @import("backend.zig"); -const Backend = backend.Backend; -const buffered_loader = @import("buffered_loader.zig"); -const LoadPart = backend.LoadPart; -const LoadSpec = backend.LoadSpec; -const DeviceBytes = backend.DeviceBytes; -const direct_loader = @import("direct_loader.zig"); -const load_log = @import("log.zig").load; -const ShardGrid = @import("ShardGrid.zig"); -const TensorStore = @import("TensorStore.zig"); - -const PrepareError = std.mem.Allocator.Error || error{InvalidBinding}; - -/// Loads models from their checkpoint sources. A value: `init` returns at -/// once. -/// -/// What it borrows. The platform outlives the loader and every buffer it -/// loads. `io`, and the VFS behind it, outlive the loader: its tasks run on -/// `io`, and the direct backend opens each source file through it and -/// keeps it open until `deinit`, under a copy of the file's path, so -/// nothing of the store is kept past `wait`. `start` borrows the store, the -/// buffers and the progress node until `wait` or `cancel` returns, and -/// reads the model during the call only. The loaded buffers are the -/// caller's and outlive the loader. -/// -/// Start and wait serially on the owning task; source reads and transfers -/// run concurrently inside the backend, compiles on tasks of their own, or -/// inline when `io` cannot run one concurrently. Submissions are retired in -/// publish order. -/// -/// What fails. Every error of `start` and `wait` fails the loader: -/// everything started is canceled (`cancel`), and every later call returns -/// the first error. Each is logged once, at err, where it is detected, but -/// for a plain allocation failure (`OutOfMemory` from an allocator) and -/// `Canceled`, the caller's own `cancel`, which are returned unlogged. A -/// model whose bindings are wrong is `InvalidBinding`, logged with the -/// tensor's field path, the key its source reads (none for a source that -/// reads nothing or a run inside a byte) and the rule it breaks, or for a -/// program whose output differs, the program's name and the shape it -/// computes; buffers shaped unlike the model (`mem.pairBufferized`) too, -/// logged with the field path where they differ. The other errors a caller -/// can act on: a missing checkpoint file is `FileNotFound`; a denied one -/// `AccessDenied` locally and `PermissionDenied` remotely; a file that ends -/// inside a tensor `UnexpectedEndOfFile`, because it changed after its -/// header was read; host memory the backend cannot allocate, pin or fit -/// under its mapped ceiling `OutOfMemory`; device memory that runs out -/// PJRT's `ResourceExhausted`; an io that cannot run the loader's tasks -/// concurrently `ConcurrencyUnavailable`; `cancel` `Canceled`. The rest -/// (PJRT, the compiler, a remote that failed) passes through as its source -/// names it. -pub const Loader = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - /// `init`'s task, which warms the device allocators and creates the - /// backend, until the first `start` awaits it. - setup: ?std.Io.Future(anyerror!Backend), - /// What `setup` created; null before the first `start`, and after a - /// failed set-up. - backend: ?Backend = null, - /// `Options.program_group_bytes`. - program_group_bytes: usize, - /// The programs of the loads started since the last `wait`, each - /// compiled once. - programs: std.ArrayList(*CompiledProgram) = .empty, - /// Program groups started and not yet published, oldest first. - queued: std.Deque(Group) = .empty, - /// Submissions published and not yet retired, oldest first. - pending: std.Deque(PendingSubmission) = .empty, - /// The first error; every later call returns it. - failure: ?anyerror = null, - /// The direct backend when admission can measure the room: it counts - /// its allocations and every device reported an allocator limit at the - /// first `start`. Otherwise one program group is in flight at a time. - measured: ?*direct_loader.Loader = null, - stats: Stats = .{}, - - pub const Options = struct { - /// The checkpoint's path or URI, read by `init` only, which copies a - /// local one the VFS may read direct for the set-up task - /// (`backend.Options.checkpoint`). When `io` is a VFS, the VFS - /// backend serving it sizes the reads (`VFS.readHints`), and the - /// direct backend decides at set-up how such a local one's files - /// read, and places its pinned memory for that - /// (`direct_loader.Loader.create`). Null, or a plain `io`: local - /// sizing, pinned memory near the devices. - checkpoint: ?[]const u8 = null, - /// Concurrent source reads, 1 to `max_read_parallelism`, fixed for - /// the load. Null: 16 for a local checkpoint, 32 for a high-latency - /// one. The direct backend may lower it to fit its pinned memory - /// (logged at debug). - read_parallelism: ?usize = null, - /// The direct backend's DMA block, at most 32 MiB, and a multiple of - /// `VFS.direct_io_alignment` when `io` is a VFS and `direct_io` is - /// not `off`. Null: calibration picks one of - /// `dma_calibration.default_block_sizes`, which suits every measured - /// host. A tuning knob, ignored by the buffered backend; `init` - /// refuses a block the checkpoint's reads cannot use. - dma_block_size: ?usize = null, - /// Direct I/O for local source files, decided once per file per - /// loader (`VFS.DirectIo` for the page-cache trade-off). Nothing - /// changes when `io` is not a VFS. - direct_io: VFS.DirectIo = .auto, - /// Output bytes of the program tensors one group holds at most, one - /// tensor at least. A group's inputs are read together and freed - /// once its programs ran. - program_group_bytes: usize = 8 << 30, - - /// The default read widths by latency class, from the recorded width - /// sweeps. Local sources knee at 12 to 16 on every host measured: the - /// B70 peaks at 12 to 16 and loses 11% at 24, one MI300X prefers 12, - /// the RTX 5090 host 16 over 12, and the GB300 plateau from 12 to 48 - /// has 16 within 5% of its best rung, so 16 is the width with the - /// smallest regret. Remote sources plateau from 24 to 32: hf:// reads - /// 32 and 64 alike and AWS was flat from 24, while pinned memory grows - /// with the width (32 x 32 MiB requests pin 1.5 GiB). - const local_read_parallelism: usize = 16; - const high_latency_read_parallelism: usize = 32; - }; - - /// Concurrent source reads at most (`Options.read_parallelism`). - pub const max_read_parallelism: usize = 128; - - /// Checks `opts` and returns at once. Warming the device allocators and - /// creating the backend (for the direct one: DMA calibration and the - /// pinned pre-growth) run on a task of the loader's own, or here when - /// `io` has no concurrency; the first `start` waits for that task and - /// returns its error. A read width or a block size out of range, a - /// block that is not a multiple of the direct-I/O alignment - /// (`Options.dma_block_size`), a checkpoint whose scheme the VFS does - /// not serve, a block calibration may select whose request (the read - /// chunk rounded up to whole blocks) passes 32 MiB, or a request too - /// short for a widened direct read is `InvalidOptions`, logged. A copy - /// of the path of a local checkpoint the VFS may read direct - /// (`Options.checkpoint`) that cannot be allocated is `OutOfMemory`, - /// unlogged. - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Options, - ) error{ InvalidOptions, OutOfMemory }!Loader { - var backend_opts = try backendOptions(io, opts); - if (backend_opts.checkpoint) |path| backend_opts.checkpoint = try allocator.dupe(u8, path); - const setup_args = .{ allocator, io, platform, backend_opts }; - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .setup = io.concurrent(setUp, setup_args) catch io.async(setUp, setup_args), - .program_group_bytes = opts.program_group_bytes, - }; - } - - /// `init`'s task: the device allocators warmed, then the backend. It - /// owns `opts.checkpoint`, `init`'s copy, and runs to its end even when - /// `deinit` cancels it. - fn setUp(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: backend.Options) anyerror!Backend { - defer if (opts.checkpoint) |path| allocator.free(path); - platform.warmupDeviceAllocators(io) catch |err| { - load_log.failure(err, "warm the device allocators", .{}); - return err; - }; - return Backend.init(allocator, io, platform, opts); - } - - /// Waits for `init`'s task the first time: its backend, then whether - /// admission can measure the devices; or its error, for the caller to - /// fail the loader with. - fn awaitSetup(self: *Loader) !void { - const setup = if (self.setup) |*task| task else return; - const result = setup.await(self.io); - self.setup = null; - self.backend = try result; - self.measured = self.probeMemory(); - } - - /// Gives up on everything started: drops the queued groups, aborts the - /// backend (queued reads are dropped, reads in flight finish), retires - /// the pending submissions without running their programs, and waits - /// for the compiles. Each buffer of the started loads is then empty or - /// holds its shards, whose contents are unspecified (unwritten, errored, - /// or partly written); read none, and free the tree as after a load - /// (`mem.deinitBufferized`). The loader is failed afterwards. For an - /// error between `start` and `wait`; nothing to do when nothing was - /// started. - pub fn cancel(self: *Loader) void { - if (self.loadsStarted()) self.fail(error.Canceled); - } - - /// Cancels `init`'s task when no `start` awaited it, destroying what it - /// built, and destroys the backend: its host memory, its workers and the - /// files it keeps open, closed through `io`. The buffers it loaded stay - /// the caller's. Panics on loads neither waited for nor canceled. - pub fn deinit(self: *Loader) void { - if (self.loadsStarted()) std.debug.panic("Loader.deinit with loads started, whose store and buffers may be freed already: add `errdefer loader.cancel();` after `start`", .{}); - self.stats.logAdmission(self.measured != null); - self.queued.deinit(self.allocator); - self.pending.deinit(self.allocator); - self.programs.deinit(self.allocator); - if (self.setup) |*setup| { - if (setup.cancel(self.io)) |backend_| backend_.destroy() else |_| {} - } - if (self.backend) |backend_| backend_.destroy(); - self.* = undefined; - } - - /// Whether loads are started and neither waited for nor canceled. - fn loadsStarted(self: *const Loader) bool { - return self.queued.len != 0 or self.pending.len != 0 or self.programs.items.len != 0; - } - - /// The direct backend when every device reports an allocator limit - /// (`measured`). Decided once: the CPU plugin logs an unimplemented - /// warning on every stats call, so admission never asks a device that - /// answered nothing. - fn probeMemory(self: *const Loader) ?*direct_loader.Loader { - const direct = switch (self.backend.?) { - .direct => |direct| direct, - .buffered => return null, - }; - for (self.platform.devices) |device| { - if (device.memoryStats().bytes_limit == null) return null; - } - return direct; - } - - /// Starts loading every tensor of `model` into its buffer in `buffers`, - /// as `store` binds it, and returns; `wait` returns once it is done. - /// Each tensor is placed by its shape's sharding (replicated without a - /// mesh), into the memory its binding names, assembled from the pieces - /// of its source (`Source.Placed.bind`) with every byte no piece covers - /// set to zero. All of these are published here, as one submission. A - /// tensor a program computes (`Source.bindProgram`) starts its program - /// compiling, once for every tensor of an equal program over the same - /// placed shapes; `wait` lands its inputs and runs it. - /// - /// Every tensor and program input is checked once, before anything is - /// published: a tensor `store` did not bind, a source that reads no - /// bytes (an empty checkpoint tensor among them), a source that this - /// loader cannot assemble (a run that starts or ends inside a byte, - /// strided parts into host memory, a slice along an inner axis, a - /// broadcast or a transpose whose source range does not fit the direct - /// backend's smallest read: the checkpoint's read chunk (8 MiB locally), - /// rounded up to whole blocks of the smallest it may pick, 2 MiB or a - /// pinned `dma_block_size`), a sub-byte last dimension that does not - /// pack into whole bytes, a placement its mesh does not divide or on - /// devices the platform lacks, and a host kind the platform cannot take - /// are `error.InvalidBinding`. - /// - /// `buffers` is `mem.bufferize` of `model`, whose buffers are empty. A - /// tree of another structure (other union arms, optionals or slice - /// lengths: `mem.pairBufferized`) is `InvalidBinding`, logged with the - /// field path, before any buffer is written. `start` empties them - /// again, and each stays empty until its load writes it whole. So - /// after any error each buffer is empty or holds shards for the caller - /// to free: `mem.deinitBufferized` frees the tree either way. `buffers` - /// and `store` are borrowed until `wait` or `cancel` returns; do not end - /// `progress` before then. `start` adds one item to `progress` for each - /// tensor the loader fills, a composite one once, and for each program - /// input; the backend completes them. - /// Several loads may be started before one `wait`, from one store or - /// several. The first `start` waits for `init`'s task, and fails with - /// its error. Each `start` then gives the device memory freed before it - /// back to the devices (`Platform.releaseFreedDeviceMemory`). - pub fn start( - self: *Loader, - model: anytype, - buffers: *Bufferized(@TypeOf(model.*)), - store: *const TensorStore, - progress: std.Progress.Node, - ) !void { - if (self.failure) |err| return err; - self.startModel(model, buffers, store, progress) catch |err| { - self.fail(err); - return err; - }; - } - - fn startModel( - self: *Loader, - model: anytype, - buffers: *Bufferized(@TypeOf(model.*)), - store: *const TensorStore, - progress: std.Progress.Node, - ) !void { - try self.awaitSetup(); - // Give back what was freed before this load (compile scratch, the - // set-up's calibration buffers), so that admission's room does not - // count it and the load's peak does not include it. - self.platform.releaseFreedDeviceMemory(); - // The load's time starts once the set-up is done and the freed - // memory given back. - self.stats.begin(self.io); - var prepared = try prepareModel(self.allocator, self.platform, self.backend.?, store, model, buffers); - errdefer prepared.arena.deinit(); - var items = prepared.specs.len; - for (prepared.computed) |item| items += item.compute.inputs.len; - progress.increaseEstimatedTotalItems(items); - // Queued first: their programs compile while the rest is read. - try self.queueGroups(store, prepared.computed, progress); - if (prepared.specs.len == 0) { - prepared.arena.deinit(); - return; - } - try self.submit(prepared.specs, prepared.arena, &.{}, @splat(0), progress); - } - - /// Publishes the queued program groups, oldest first, each once its - /// programs compiled and the devices have room for it (`admit`), and - /// retires every submission in publish order, running a group's - /// programs on this task once its inputs landed. Returns once every - /// started load is in its buffers, with the freed device memory, its - /// programs' inputs and temporaries among it, given back to the devices - /// (`Platform.releaseFreedDeviceMemory`), and logs one info line: the - /// bytes loaded, the time since the first `start` and their rate, then - /// what the programs computed and how long publishing waited for their - /// compiles. A program that fails to compile or run fails it; one that - /// does not return its tensor's shape, placed alike, is - /// `InvalidBinding`; a group that does not fit a device fails it with - /// the device's error. Idempotent: with nothing started it returns at - /// once and logs nothing. - pub fn wait(self: *Loader) !void { - if (self.failure) |err| return err; - const started = self.loadsStarted(); - self.drain() catch |err| { - self.fail(err); - return err; - }; - self.releasePrograms(); - if (started) self.platform.releaseFreedDeviceMemory(); - } - - fn drain(self: *Loader) !void { - var groups: usize = 0; - var tensors: usize = 0; - while (self.queued.popFront()) |group| { - tensors += group.calls.len; - try self.publish(group); - groups += 1; - } - while (self.pending.len != 0) try self.retireOldest(true); - self.stats.end(self.io, tensors, self.programs.items.len, groups); - } - - /// Records `err` unless the loader failed already, and gives up on - /// everything started (`cancel`). The one writer of `failure`. - fn fail(self: *Loader, err: anyerror) void { - if (self.failure == null) self.failure = err; - while (self.queued.popFront()) |group| group.deinit(self.allocator); - // Nothing still pending can succeed. - if (self.pending.len != 0) self.backend.?.abort(); - while (self.pending.len != 0) self.retireOldest(false) catch {}; - self.releasePrograms(); - } - - /// Queues the tensors of `computed`, in model order, in groups of at - /// most `program_group_bytes` of outputs, one tensor at least. - fn queueGroups(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) !void { - var first: usize = 0; - while (first < computed.len) { - var end = first + 1; - var bytes = computed[first].shape.byteSize(); - while (end < computed.len and bytes + computed[end].shape.byteSize() <= self.program_group_bytes) : (end += 1) { - bytes += computed[end].shape.byteSize(); - } - try self.queued.ensureUnusedCapacity(self.allocator, 1); - self.queued.pushBackAssumeCapacity(try self.prepareGroup(store, computed[first..end], progress)); - first = end; - } - } - - /// The group computing `computed`: each tensor's program, the input - /// shells it runs over and the specs that fill them, one per input, - /// checked as `start` checks every spec (`prepareSpec`) before the - /// program's compile starts, which packs their shapes. - fn prepareGroup(self: *Loader, store: *const TensorStore, computed: []const Computed, progress: std.Progress.Node) PrepareError!Group { - var arena: std.heap.ArenaAllocator = .init(self.allocator); - errdefer arena.deinit(); - const calls = try self.allocator.alloc(Call, computed.len); - var prepared: usize = 0; - errdefer { - for (calls[0..prepared]) |*call| call.deinit(self.allocator); - self.allocator.free(calls); - } - var specs: std.ArrayList(LoadSpec) = .empty; - for (computed, calls) |item, *call| { - const inputs = try self.allocator.alloc(Buffer, item.compute.inputs.len); - errdefer self.allocator.free(inputs); - for (item.compute.inputs, inputs, 0..) |source, *input, index| { - input.* = .{ ._platform = self.platform, ._shape = source.shape, ._shards = .empty }; - const target: Target = .{ .path = item.path, .shape = item.shape, .input = .{ .index = index, .shape = source.shape } }; - const spec = try prepareSpec(arena.allocator(), self.platform, self.backend.?, target, source, .default, input); - try specs.append(arena.allocator(), spec); - } - const compiled = try self.compiledProgram(store, item.compute, item.shape); - call.* = .{ .path = try arena.allocator().dupe(u8, item.path), .program = compiled, .inputs = inputs, .output = item.output }; - prepared += 1; - } - return .{ .calls = calls, .specs = specs.items, .arena = arena, .progress = progress }; - } - - /// The compiled program computing a tensor of `output` from `compute`: - /// one started already for an equal program over the same placed - /// shapes, else a compile started now. - fn compiledProgram(self: *Loader, store: *const TensorStore, compute: TensorStore.Binding.Compute, output: Shape) std.mem.Allocator.Error!*CompiledProgram { - for (self.programs.items) |compiled| { - if (compiled.compute.sameExecutable(compiled.output, compute, output)) return compiled; - } - try self.programs.ensureUnusedCapacity(self.allocator, 1); - const compiled = try CompiledProgram.start(self.allocator, self.io, self.platform, store.meshes, compute, output); - self.programs.appendAssumeCapacity(compiled); - return compiled; - } - - fn releasePrograms(self: *Loader) void { - for (self.programs.items) |compiled| compiled.destroy(self.allocator, self.io); - self.programs.clearRetainingCapacity(); - } - - /// Publishes `group` once its programs compiled and the devices have - /// room for it: its calls and its specs' arena move to the pending - /// submission, or are freed when this fails. - fn publish(self: *Loader, group: Group) !void { - errdefer group.deinit(self.allocator); - const started: std.Io.Timestamp = .now(self.io, .awake); - for (group.calls) |call| try call.program.await(self.io, self.platform); - self.stats.compile_wait_ns +|= @intCast(started.untilNow(self.io, .awake).toNanoseconds()); - const execution = try self.admit(group.calls); - try self.submit(group.specs, group.arena, group.calls, execution, group.progress); - } - - /// Retires pending submissions, oldest first, until the group of - /// `calls` fits the measured room or no program group is pending: a - /// bulk load frees nothing. Once nothing can be retired for it, the - /// room is read again after the freed device memory is given back - /// (`Platform.releaseFreedDeviceMemory`), and the group is admitted - /// whatever its size (`Stats.beyond_room`). Without memory measurement - /// one program group is in flight at a time. Returns the group's output - /// and temporary bytes per device, zero while unmeasured. - fn admit(self: *Loader, calls: []const Call) !DeviceBytes { - var inputs: DeviceBytes = @splat(0); - var execution: DeviceBytes = @splat(0); - var room: ?DeviceBytes = null; - if (self.measured) |direct| { - var temporaries: DeviceBytes = @splat(0); - for (calls) |call| { - const exe = &call.program.exe.?; - for (exe.input_shapes) |shape| addPlacementBytes(&inputs, self.platform, shape); - // The output's memory space is the program's to choose; it is - // charged to the devices like the default space. - addPlacementBytes(&execution, self.platform, exe.output_shapes[0]); - // The group's programs run one after the other, so only the - // largest temporaries are live at once. - const temp = self.compiledTempBytes(exe); - const output_mesh = self.platform.placedShape(exe.output_shapes[0])._sharding.mesh.?; - for (output_mesh.devicesInCanonicalOrder()) |device| temporaries[device.id] = @max(temporaries[device.id], temp); - } - for (&execution, temporaries) |*bytes, temp| bytes.* +|= temp; - room = self.readRoom(direct); - } - var retired: usize = 0; - while (room == null or !self.fits(&room.?, &inputs, &execution)) : (retired += 1) { - if (!self.programGroupPending()) { - // Nothing is left to retire, but the retired groups' inputs - // may still count as in use: read the room again once the - // devices have them back. The group then goes, beyond the - // room if the room is still short. - if (room != null) { - self.platform.releaseFreedDeviceMemory(); - room = self.readRoom(self.measured.?); - } - break; - } - try self.retireOldest(true); - if (self.measured) |direct| room = self.readRoom(direct); - } - const beyond_room = room != null and !self.fits(&room.?, &inputs, &execution); - self.stats.admission_retires += retired; - if (beyond_room) self.stats.beyond_room += 1; - if (retired != 0 or beyond_room) { - // Unmeasured, the group's bytes are not computed: only the retires are logged. - if (self.measured != null) { - const n = self.platform.devices.len; - // A device that stopped reporting its limit shows no room. - const rooms: DeviceBytes = room orelse @splat(0); - const pending = self.pendingExecution(); - // The device the fit is decided on: the short one when the - // group goes beyond the room. - const device = admission.tightest(rooms[0..n], pending[0..n], inputs[0..n], execution[0..n]); - load_log.debug("execute admission: retired={d} before submission {d}: measured=true, beyond_room={}, device {d} room={Bi:.2} pending_execution={Bi:.2} inputs={Bi:.2} execution={Bi:.2}", .{ - retired, - self.stats.submissions, - beyond_room, - device, - rooms[device], - pending[device], - inputs[device], - execution[device], - }); - } else { - load_log.debug("execute admission: retired={d} before submission {d}: measured=false", .{ retired, self.stats.submissions }); - } - } - return execution; - } - - /// Whether a group of `inputs` and `execution` fits beside the pending - /// program groups in `room`, on every device. - fn fits(self: *const Loader, room: *const DeviceBytes, inputs: *const DeviceBytes, execution: *const DeviceBytes) bool { - const n = self.platform.devices.len; - const pending = self.pendingExecution(); - return admission.admits(room[0..n], pending[0..n], inputs[0..n], execution[0..n]); - } - - /// Output and temporary bytes of the pending program groups, per - /// device: what retiring them will take before it frees anything. - fn pendingExecution(self: *const Loader) DeviceBytes { - var sum: DeviceBytes = @splat(0); - var it = self.pending.iterator(); - while (it.nextPtr()) |submission| { - for (&sum, submission.execution) |*bytes, more| bytes.* +|= more; - } - return sum; - } - - /// Whether a program group is pending: only retiring one frees room - /// for the next (a bulk load's buffers stay). - fn programGroupPending(self: *const Loader) bool { - var it = self.pending.iterator(); - while (it.nextPtr()) |submission| { - if (submission.calls.len != 0) return true; - } - return false; - } - - /// The room each device reports now; null when one reports no memory - /// limit. - fn readRoom(self: *Loader, direct: *const direct_loader.Loader) ?DeviceBytes { - const n = self.platform.devices.len; - const unallocated = direct.unallocatedBytes(); - var room: DeviceBytes = undefined; - for (room[0..n], self.platform.devices, unallocated[0..n]) |*out, device, device_unallocated| { - const reported = device.memoryStats(); - out.* = admission.room(reported.bytes_limit, reported.bytes_in_use, device_unallocated, admission.reserve_bytes) orelse return null; - self.stats.min_room_seen = @min(self.stats.min_room_seen orelse out.*, out.*); - } - return room; - } - - /// Temporaries the executable takes on each of its devices, from its - /// compiled memory stats (per partition, i.e. per device). Zero when - /// the plugin cannot answer. - fn compiledTempBytes(self: *Loader, exe: *const Exe) u64 { - const api = self.platform.pjrt_api; - const pjrt_executable = exe.exe.executable(api) catch |err| return self.tempUnavailable(err); - defer pjrt_executable.deinit(api); - const stats = pjrt_executable.getCompiledMemoryStats(api) catch |err| return self.tempUnavailable(err); - return stats.temp_size_in_bytes; - } - - fn tempUnavailable(self: *Loader, err: pjrt.ApiError) u64 { - if (!self.stats.temp_unavailable_logged) { - self.stats.temp_unavailable_logged = true; - load_log.debug("executable memory stats unavailable ({s}): temporaries counted as zero", .{@errorName(err)}); - } - return 0; - } - - /// One submission over `specs`, which live in `arena`. Takes `arena` and - /// `calls` once the submission is published; on failure the caller still - /// owns them. `execution` is a program group's output and temporary - /// bytes per device (`admit`), zero for a bulk load. - fn submit( - self: *Loader, - specs: []const LoadSpec, - arena: std.heap.ArenaAllocator, - calls: []Call, - execution: DeviceBytes, - progress: std.Progress.Node, - ) !void { - // One spec per destination, however many parts fill it: its bytes - // count once. - var logical_bytes: usize = 0; - for (specs) |spec| logical_bytes += spec.shape.byteSize(); - try self.pending.ensureUnusedCapacity(self.allocator, 1); - const submission = try self.backend.?.submit(specs, progress); - self.pending.pushBackAssumeCapacity(.{ - .submission = submission, - .arena = arena, - .calls = calls, - .execution = execution, - .logical_bytes = logical_bytes, - }); - self.stats.submissions += 1; - } - - /// Retires the oldest pending submission: waits for its reads and - /// transfers, runs its programs when `execute`, frees its inputs either - /// way and counts its bytes once they ran. - fn retireOldest(self: *Loader, execute: bool) !void { - var oldest = self.pending.popFront().?; - defer self.release(&oldest); - try self.backend.?.await(oldest.submission); - if (!execute) return; - for (oldest.calls) |*call| { - call.execute(self.allocator, self.io) catch |err| { - const target: Target = .{ .path = call.path, .shape = call.program.output }; - load_log.failure(err, "run program {s} computing {f}: inputs={d}", .{ call.program.name(), target, call.inputs.len }); - return err; - }; - } - self.stats.bytes_loaded += oldest.logical_bytes; - } - - fn release(self: *Loader, oldest: *PendingSubmission) void { - oldest.arena.deinit(); - for (oldest.calls) |*call| call.deinit(self.allocator); - self.allocator.free(oldest.calls); - } -}; - -/// The loader's counters: read only by logs and tests; nothing here -/// decides. -const Stats = struct { - /// When the first `start` since the last `wait` ran: `wait` logs the - /// load's time from there. - started_at: ?std.Io.Timestamp = null, - /// Logical bytes of the submissions retired with execution since that - /// `start`: each destination's shape once, so a composite tensor counts - /// its padding and not its sources one by one, and a program group - /// counts its inputs. - bytes_loaded: usize = 0, - /// How long publishing waited for compiles since the last `wait`. - compile_wait_ns: u64 = 0, - /// Submissions published since `init`. - submissions: usize = 0, - /// Submissions retired to admit a program group, bulk loads included. - admission_retires: usize = 0, - /// The least room a device reported since `init`; null while nothing - /// was measured. - min_room_seen: ?u64 = null, - /// Program groups admitted beyond the measured room since `init`: - /// nothing was left to retire for them. - beyond_room: usize = 0, - /// Whether `tempUnavailable` logged that the plugin gives no compiled - /// memory stats: once per loader. - temp_unavailable_logged: bool = false, - - /// Times and counts a load from the first `start` since the last - /// `wait`; later starts add to it. - fn begin(self: *Stats, io: std.Io) void { - if (self.started_at != null) return; - self.started_at = .now(io, .awake); - self.bytes_loaded = 0; - } - - /// Logs `wait`'s info line, unless nothing was started, and ends the - /// load's timing. `programs` programs computed `tensors` tensors in - /// `groups` groups. - fn end(self: *Stats, io: std.Io, tensors: usize, programs: usize, groups: usize) void { - const started_at = self.started_at orelse return; - const took = started_at.untilNow(io, .awake); - const seconds = @as(f64, @floatFromInt(@max(took.nanoseconds, 1))) / std.time.ns_per_s; - const rate: u64 = @intFromFloat(@as(f64, @floatFromInt(self.bytes_loaded)) / seconds); - if (groups == 0) { - load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ self.bytes_loaded, took, rate }); - } else { - load_log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s], computed {d} tensors with {d} programs in {d} groups, {d} ms waiting for compiles", .{ - self.bytes_loaded, - took, - rate, - tensors, - programs, - groups, - self.compile_wait_ns / std.time.ns_per_ms, - }); - } - self.started_at = null; - self.compile_wait_ns = 0; - } - - /// `deinit`'s debug line. `measured`: whether admission measured the - /// room (`Loader.measured`), printed as `memory_supported`. - fn logAdmission(self: *const Stats, measured: bool) void { - if (self.submissions == 0) return; - load_log.debug("loader admission: submissions={d}, execute_admission_retires={d}, beyond_room={d}, memory_supported={}, min_room_seen={Bi:.2}, reserve={Bi:.2}", .{ - self.submissions, - self.admission_retires, - self.beyond_room, - measured, - self.min_room_seen orelse 0, - admission.reserve_bytes, - }); - } -}; - -/// One published submission: what to wait for, what to run afterwards and -/// what its accounting owes. -const PendingSubmission = struct { - submission: backend.Submission, - /// The specs, which the backend reads until the submission is awaited - /// (`backend.LoadSpec`). - arena: std.heap.ArenaAllocator, - /// Run in order once the reads are done; freed with their inputs when - /// the submission is retired. Empty for a bulk load. - calls: []Call, - /// Output and temporary bytes per device of a program group, while - /// admission measures (`admit`); zero otherwise and for a bulk load. - execution: DeviceBytes, - logical_bytes: usize, -}; - -/// Program tensors whose inputs `wait` publishes as one submission. -const Group = struct { - calls: []Call, - /// One per program input, filling the calls' input shells; they and - /// their parts live in `arena`. - specs: []LoadSpec, - arena: std.heap.ArenaAllocator, - progress: std.Progress.Node, - - fn deinit(self: Group, allocator: std.mem.Allocator) void { - for (self.calls) |*call| call.deinit(allocator); - allocator.free(self.calls); - self.arena.deinit(); - } -}; - -/// A program compiling, then compiled. -const CompiledProgram = struct { - compute: TensorStore.Binding.Compute, - output: Shape, - future: std.Io.Future(Compiler.Error!Exe), - /// Set by the first `await`. - exe: ?Exe = null, - - fn start( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - meshes: []const *const Sharding.Mesh, - compute: TensorStore.Binding.Compute, - output: Shape, - ) std.mem.Allocator.Error!*CompiledProgram { - const self = try allocator.create(CompiledProgram); - // Concurrent, as `io.async` may run the compile right here on a busy - // threaded io, ahead of the reads `start` publishes after it. - const args = .{ compute, allocator, io, platform, meshes, output }; - const compile = TensorStore.Binding.Compute.compile; - self.* = .{ - .compute = compute, - .output = output, - .future = io.concurrent(compile, args) catch io.async(compile, args), - }; - return self; - } - - /// Waits for the compile the first time, then checks that the - /// executable computes one tensor of `output`, placed alike - /// (`checkOutput`). - fn await(self: *CompiledProgram, io: std.Io, platform: *const Platform) !void { - if (self.exe != null) return; - self.exe = self.future.await(io) catch |err| { - load_log.failure(err, "compile program {s} computing {f}", .{ self.name(), self.output }); - return err; - }; - try checkOutput(platform, self.name(), self.output, &self.exe.?); - } - - fn name(self: *const CompiledProgram) []const u8 { - return self.compute.program.name(); - } - - fn destroy(self: *CompiledProgram, allocator: std.mem.Allocator, io: std.Io) void { - if (self.exe) |*exe| { - exe.deinit(); - } else if (self.future.cancel(io)) |exe| { - exe.deinit(); - } else |_| {} - allocator.destroy(self); - } -}; - -/// One tensor of a program group, a pending `Exe.call`: its program, the -/// input shells the group's submission fills, and the model's buffer it -/// computes. -const Call = struct { - /// The tensor's field path, in the group's arena: `prepareModel`'s - /// arena may be freed before the call runs. - path: []const u8, - program: *CompiledProgram, - inputs: []Buffer, - output: *Buffer, - - /// Runs the program's executable, awaited when the group was - /// published, over the loaded inputs on the calling task. - /// `Exe.call` leaves every input handle to the caller, donated ones - /// included (their contents are gone), so `deinit` frees them all, - /// exactly once; a runner would destroy the donated handles itself. - /// A failed execution, an input whose transfer failed included, is - /// returned and leaves `output` unwritten. - fn execute(self: *const Call, allocator: std.mem.Allocator, io: std.Io) !void { - const exe = &self.program.exe.?; - var args = try exe.args(allocator); - defer args.deinit(allocator); - var results = try exe.results(allocator); - defer results.deinit(allocator); - args.set(.{self.inputs}); - try exe.call(io, args, &results, .{ .wait = true }); - self.output.* = results.get(Buffer); - } - - /// Frees the inputs, donated or not; a shell the loader never wrote owns - /// no shards. - fn deinit(self: *Call, allocator: std.mem.Allocator) void { - for (self.inputs) |*input| input.deinit(); - allocator.free(self.inputs); - } -}; - -/// The arithmetic behind program admission: what the devices have left, -/// and whether the next group fits beside the ones still pending. -/// Everything is per device (index = `platform.devices` index) in bytes; -/// nothing here touches a device, so the rules are tested without one. -const admission = struct { - /// Allocator slack per device. BFC rounds every allocation up to 256 bytes; - /// a 64-pack, 16-source, 8-device load is about 1e5 shards, under 32 MiB of - /// rounding, and nothing else runs on the devices during a load. - const reserve_bytes: u64 = 64 << 20; - - /// Bytes the loader may still take on one device: the limit (null when the - /// plugin reports none, as CPU does) minus what is in use, minus its own - /// bytes published but not allocated yet (`unallocated`), minus the - /// reserve. - fn room(limit: ?u64, in_use: u64, unallocated: u64, reserve: u64) ?u64 { - return (limit orelse return null) -| in_use -| unallocated -| reserve; - } - - /// Whether one submission's `inputs` (placements, counted here until they - /// are published) and `execution` (output placement plus the executable's - /// temporaries) fit beside the pending executions on every device. - fn admits(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) bool { - for (rooms, pending, inputs, execution) |r, p, i, e| { - if (p +| i +| e > r) return false; - } - return true; - } - - /// The device with the least room left once one submission's `inputs` - /// and `execution` sit beside the pending executions: one that `admits` - /// fails on when the submission does not fit. `rooms` is not empty. - fn tightest(rooms: []const u64, pending: []const u64, inputs: []const u64, execution: []const u64) usize { - var device: usize = 0; - var least: i128 = std.math.maxInt(i128); - for (rooms, pending, inputs, execution, 0..) |r, p, i, e, index| { - const left = @as(i128, r) - (p +| i +| e); - if (left < least) { - least = left; - device = index; - } - } - return device; - } -}; - -test "room subtracts use, unallocated bytes and the reserve, saturating" { - try std.testing.expectEqual(null, admission.room(null, 0, 0, 0)); - try std.testing.expectEqual(55, admission.room(100, 30, 10, 5).?); - // More in use than the limit admits: no room, no underflow. - try std.testing.expectEqual(0, admission.room(100, 130, 0, 0).?); - try std.testing.expectEqual(0, admission.room(100, 90, 0, 20).?); -} - -test "admits needs room on every device" { - try std.testing.expect(admission.admits(&.{ 40, 40 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); - // The second device is one byte short. - try std.testing.expect(!admission.admits(&.{ 40, 34 }, &.{ 20, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); - try std.testing.expect(admission.admits(&.{}, &.{}, &.{}, &.{})); -} - -test "tightest names the device short of room, not the first" { - // Device 0 has less room but holds the group; device 1 is one byte short. - try std.testing.expectEqual(1, admission.tightest(&.{ 30, 34 }, &.{ 0, 20 }, &.{ 10, 10 }, &.{ 5, 5 })); - // Both are short: the one short by more, though it has more room. - try std.testing.expectEqual(1, admission.tightest(&.{ 10, 12 }, &.{ 0, 0 }, &.{ 10, 20 }, &.{ 5, 10 })); - // A need that saturates at maxInt(u64) is short by the most. - try std.testing.expectEqual(1, admission.tightest(&.{ 10, 20 }, &.{ 0, std.math.maxInt(u64) }, &.{ 20, 1 }, &.{ 0, 0 })); - // Equally short: the first. - try std.testing.expectEqual(0, admission.tightest(&.{ 10, 10 }, &.{ 0, 0 }, &.{ 12, 12 }, &.{ 0, 0 })); -} - -/// Adds the bytes `shape` occupies per device: its box on each device it -/// is placed on, as the direct backend's `countSealed` counts them; an -/// empty shape takes none. `shape` places on the platform's devices -/// (`checkPlacement`). -fn addPlacementBytes(out: *DeviceBytes, platform: *const Platform, shape: Shape) void { - if (shape.byteSize() == 0) return; - const grid: ShardGrid = .init(platform, shape); - for (grid.devices) |device| out[device.id] += grid.box_bytes; -} - -/// A tensor a program computes, at `path` in the model, the buffer it -/// computes into, and its program and inputs. -const Computed = struct { - path: []const u8, - shape: Shape, - output: *Buffer, - compute: TensorStore.Binding.Compute, -}; - -/// A model as `start` takes it: one spec per tensor the loader fills, and -/// the tensors programs compute, all in `arena` with the specs' parts and -/// the tensors' field paths. -const PreparedModel = struct { - arena: std.heap.ArenaAllocator, - specs: []const LoadSpec, - computed: []const Computed, -}; - -/// Pairs each tensor of `model` with its buffer and binding: a tensor the -/// loader fills takes one spec (`prepareSpec`), a tensor a program computes -/// is placed like any other (`checkPlacement`) and computed. -fn prepareModel( - allocator: std.mem.Allocator, - platform: *const Platform, - backend_: Backend, - store: *const TensorStore, - model: anytype, - buffers: *Bufferized(@TypeOf(model.*)), -) PrepareError!PreparedModel { - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - // Tensors and buffers pair by structure, before any buffer is written: a - // buffer tree shaped otherwise is refused whole. - var report_buffer: [256]u8 = undefined; - var report: std.Io.Writer = .fixed(&report_buffer); - const pairs = mem.pairBufferized(arena.allocator(), model, buffers, &report) catch |err| switch (err) { - error.Mismatch => { - load_log.err("buffers shaped unlike the model at model{s}; bufferize them from this model", .{report.buffered()}); - return error.InvalidBinding; - }, - error.OutOfMemory => return error.OutOfMemory, - }; - // Empty until a backend or a program writes it whole, so the caller can - // free the tree whatever fails. - for (pairs) |pair| pair.buffer.* = .{ ._platform = platform, ._shape = pair.tensor.shape(), ._shards = .empty }; - - var specs: std.ArrayList(LoadSpec) = try .initCapacity(arena.allocator(), pairs.len); - var computed: std.ArrayList(Computed) = try .initCapacity(arena.allocator(), pairs.len); - for (pairs) |pair| { - const tensor = pair.tensor; - const target: Target = .{ .path = pair.path, .shape = tensor.shape() }; - const binding = store.bindingOf(tensor.id) orelse { - load_log.err("{f} has no checkpoint binding", .{target}); - return error.InvalidBinding; - }; - switch (binding) { - .load => |load| specs.appendAssumeCapacity(try prepareSpec(arena.allocator(), platform, backend_, target, load.source, load.memory, pair.buffer)), - .compute => |compute| { - try checkPlacement(platform, target, compute.program.name()); - computed.appendAssumeCapacity(.{ .path = pair.path, .shape = tensor.shape(), .output = pair.buffer, .compute = compute }); - }, - } - } - return .{ .arena = arena, .specs = specs.items, .computed = computed.items }; -} - -/// A strided part with few runs, or with large ones, expands into one part -/// per run, each read and transferred on its own: at most `max_part_runs` -/// runs, or runs of at least `large_run_bytes`. A spec with any other -/// strided part goes to the backend whole, as strided parts -/// (`backend.Lattice`), which the direct backend reads by destination -/// ranges. -const max_part_runs = 64; -const large_run_bytes = 1 << 20; - -fn expandable(lattice: backend.Lattice) bool { - return lattice.dims.len == 0 or lattice.runs() <= max_part_runs or lattice.width >= large_run_bytes; -} - -/// What a spec fills, as the logs name it: the model's tensor at `path` -/// (`mem.Pair.path`) of `shape`, or `input` of the program computing it. -const Target = struct { - path: []const u8, - shape: Shape, - input: ?Input = null, - - const Input = struct { index: usize, shape: Shape }; - - /// The shape the spec fills: the input's, else the tensor's. - fn destination(self: Target) Shape { - return if (self.input) |input| input.shape else self.shape; - } - - pub fn format(self: Target, writer: *std.Io.Writer) !void { - if (self.input) |input| try writer.print("program input {d} {f} of ", .{ input.index, input.shape }); - try writer.print("model{s} {f}", .{ self.path, self.shape }); - } -}; - -/// The spec filling `output`, the buffer of `target`, from `source` into -/// `memory`, its parts in `arena`: the source lowered to bytes -/// (`Source.Placed.lower`), expanded into its runs unless a part does not -/// expand, in destination order. The one check of what the backends trust -/// (`backend.LoadSpec`, `Backend.stridedRead`); a rule it breaks is -/// `InvalidBinding`, logged with the target, and with the key its source -/// reads once the source is lowered. -fn prepareSpec( - arena: std.mem.Allocator, - platform: *const Platform, - backend_: Backend, - target: Target, - source: TensorStore.Source.Placed, - memory: Memory, - output: *Buffer, -) PrepareError!LoadSpec { - const shape = target.destination(); - const lowered = source.lower(arena) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.SubBytePosition => { - load_log.err("{f}: its source {f} reads a run that starts or ends inside a byte, which the loader cannot place", .{ target, source.shape }); - return error.InvalidBinding; - }, - }; - if (lowered.len == 0) { - load_log.err("{f} reads no checkpoint bytes", .{target}); - return error.InvalidBinding; - } - const key = lowered[0].source.name; - var expands = true; - var runs: u64 = 0; - for (lowered) |part| { - expands = expands and expandable(part.lattice); - runs += part.lattice.runs(); - } - try checkPlacement(platform, target, key); - if (!backend.supportsMemory(platform, memory)) { - load_log.err("{f} from {s} is bound to {t} memory, which {t} cannot take", .{ target, key, memory, platform.target }); - return error.InvalidBinding; - } - if (!expands and memory.isHost()) { - load_log.err("{f} from {s}: strided parts into {t} memory, which this loader does not assemble", .{ target, key, memory }); - return error.InvalidBinding; - } - const parts = if (!expands) lowered else expanded: { - if (runs == lowered.len) break :expanded lowered; - const expanded = try arena.alloc(LoadPart, @intCast(runs)); - var next: usize = 0; - for (lowered) |part| { - var cursor: backend.Lattice.Cursor = .init(part.lattice, 0, part.lattice.runs()); - while (cursor.next()) |run| : (next += 1) { - expanded[next] = .{ .source = part.source, .lattice = .{ .src_offset = run.src, .dst_offset = run.dst, .width = part.lattice.width } }; - } - } - break :expanded expanded; - }; - std.mem.sort(LoadPart, parts, {}, struct { - fn lessThan(_: void, lhs: LoadPart, rhs: LoadPart) bool { - return lhs.lattice.dst_offset < rhs.lattice.dst_offset; - } - }.lessThan); - if (!backend.checkParts(parts, shape.byteSize())) { - load_log.err("{f} from {s}: its parts leave their sources or its {d} bytes, pass the largest file offset, or come out of order", .{ target, key, shape.byteSize() }); - return error.InvalidBinding; - } - // After `checkParts`, which bounds the offsets `stridedRead` adds. - for (parts) |part| { - const strided = backend_.stridedRead(part); - if (strided.read > strided.limit) { - load_log.err("{f} from {s}: a strided part spans {d} source bytes, more than one read of {d}", .{ target, key, strided.read, strided.limit }); - return error.InvalidBinding; - } - } - return .{ .shape = shape, .memory = memory, .output = output, .parts = parts }; -} - -/// Checks that `target`, from `from` (its source's key or its program), -/// places on `platform`'s devices, in any memory space: a sub-byte last -/// dimension packs into whole bytes (`Shape.packedShape`), its mesh -/// partitions divide its dimensions, and every device of its mesh is one of -/// the platform's. -fn checkPlacement(platform: *const Platform, target: Target, from: []const u8) error{InvalidBinding}!void { - const shape = target.destination(); - const bits = shape.dtype().bitSizeOf(); - if (bits < 8 and (shape.rank() == 0 or @rem(shape.dim(-1), 8 / bits) != 0)) { - load_log.err("{f} from {s}: its last dimension does not pack into whole bytes", .{ target, from }); - return error.InvalidBinding; - } - const placed = platform.placedShape(shape).packedShape(); - const placement = Sharding.Placement.init(&placed) catch { - load_log.err("{f} from {s}: its dimensions are not divisible by its mesh partitions", .{ target, from }); - return error.InvalidBinding; - }; - for (placement.mesh().?.devicesInCanonicalOrder()) |device| { - if (device.id >= platform.devices.len) { - load_log.err("{f} from {s} is placed on device {d}, but {t} has {d}", .{ target, from, device.id, platform.target, platform.devices.len }); - return error.InvalidBinding; - } - } -} - -/// `opts` as the backends take them, the checkpoint resolved to the read -/// hints of the VFS backend serving it and the read width to its default, -/// and borrowed when it is local and may read direct -/// (`backend.Options.checkpoint`). Every option is checked here, once. -fn backendOptions(io: std.Io, opts: Loader.Options) error{InvalidOptions}!backend.Options { - const hints: VFS.ReadHints = hints: { - const path = opts.checkpoint orelse break :hints .local; - const vfs = VFS.fromIo(io) orelse break :hints .local; - break :hints vfs.readHints(path) catch |err| { - load_log.err("invalid loader options: checkpoint={s}: {s}", .{ path, @errorName(err) }); - return error.InvalidOptions; - }; - }; - if (opts.read_parallelism) |width| { - if (width == 0 or width > Loader.max_read_parallelism) { - load_log.err("invalid loader options: read_parallelism={d}, expected 1..{d}: InvalidOptions", .{ width, Loader.max_read_parallelism }); - return error.InvalidOptions; - } - } - const max_request = backend.max_read_request_size; - if (hints.read_chunk_size == 0 or hints.read_chunk_size > max_request) { - load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, expected 1..{d}: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, max_request }); - return error.InvalidOptions; - } - var options: backend.Options = .{ - .read_parallelism = opts.read_parallelism orelse if (hints.high_latency) - Loader.Options.high_latency_read_parallelism - else - Loader.Options.local_read_parallelism, - .read_hints = hints, - .dma_block_size = opts.dma_block_size, - .direct_io = opts.direct_io, - }; - // Checked for every block calibration may select. - const alignment = backend.readAlignment(io, opts.direct_io); - for (options.dmaBlockSizes()) |block_size| { - if (backend.requestSize(hints.read_chunk_size, block_size, alignment) == null) { - load_log.err("invalid loader options: checkpoint={?s}, read_chunk_size={d}, dma_block_size={d}, alignment={d}, max_request={d}: no request size fits: InvalidOptions", .{ opts.checkpoint, hints.read_chunk_size, block_size, alignment, max_request }); - return error.InvalidOptions; - } - } - // A nonzero alignment means a VFS that may read a local file direct. - // `isLocal` opens nothing. - const path = opts.checkpoint orelse return options; - if (alignment != 0 and VFS.fromIo(io).?.isLocal(path)) options.checkpoint = path; - return options; -} - -/// A program must compute one tensor of `expected`, placed on the same -/// devices, each holding the same slice. The comparison is per device, not -/// by mesh pointer and partition, so `.open` and `.replicated`, or a null -/// mesh and the replicated mesh, agree when they place alike. `expected` -/// places on the platform's devices (`checkPlacement`). -fn checkOutput(platform: *const Platform, name: []const u8, expected: Shape, exe: *const Exe) error{InvalidBinding}!void { - if (exe.output_shapes.len != 1 or !expected.eql(exe.output_shapes[0])) { - load_log.err("program {s} computing {f} returns {d} outputs, the first {f}", .{ name, expected, exe.output_shapes.len, if (exe.output_shapes.len != 0) exe.output_shapes[0] else expected }); - return error.InvalidBinding; - } - const actual = exe.output_shapes[0]; - const want_shape = platform.placedShape(expected).packedShape(); - const want = Sharding.Placement.init(&want_shape) catch |err| - std.debug.panic("loader: {f} does not place, though checked: {s}", .{ expected, @errorName(err) }); - const got_shape = platform.placedShape(actual).packedShape(); - const got = Sharding.Placement.init(&got_shape) catch |err| { - load_log.err("program {s}: placement of its output {f}: {s}", .{ name, actual, @errorName(err) }); - return error.InvalidBinding; - }; - const want_devices = want.mesh().?.devicesInCanonicalOrder(); - const got_devices = got.mesh().?.devicesInCanonicalOrder(); - if (want_devices.len != got_devices.len) { - load_log.err("program {s} places its output {f} on {d} devices, the tensor {f} on {d}", .{ name, actual, got_devices.len, expected, want_devices.len }); - return error.InvalidBinding; - } - for (want_devices, got_devices, 0..) |w, g, device_index| { - const want_slices = want.slices(device_index, want_shape); - const got_slices = got.slices(device_index, got_shape); - const same_slices = want_slices.len == got_slices.len and for (want_slices.constSlice(), got_slices.constSlice()) |a, b| { - if (a.start != b.start or a.size != b.size) break false; - } else true; - if (w.id != g.id or !same_slices) { - load_log.err("program {s} places its output {f} unlike the tensor {f} on device {d}", .{ name, actual, expected, w.id }); - return error.InvalidBinding; - } - } -} - -/// A four-byte tensor `value`, a four-byte `second`, a `missing` entry in a -/// file that does not exist, an `empty` tensor and a CPU platform. Pinned -/// after `init`. -const LoaderTestFixture = struct { - const contents = [_]u8{ 1, 2, 3, 4 }; - const second_contents = [_]u8{ 5, 6, 7, 8 }; - - tmp: std.testing.TmpDir, - store: TensorStore, - platform: *const Platform, - value: Tensor, - second: Tensor, - missing: Tensor, - empty: Tensor, - - fn init(self: *LoaderTestFixture, allocator: std.mem.Allocator, io: std.Io) !void { - self.tmp = std.testing.tmpDir(.{}); - errdefer self.tmp.cleanup(); - const file = try self.tmp.dir.createFile(io, "weights.bin", .{ .read = true }); - try file.writePositionalAll(io, &(contents ++ second_contents), 0); - var path_buffer: [1024]u8 = undefined; - const path_len = try file.realPath(io, &path_buffer); - file.close(io); - const path = path_buffer[0..path_len]; - var missing_buffer: [1024]u8 = undefined; - const missing_path = try std.fmt.bufPrint(&missing_buffer, "{s}.missing", .{path}); - - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ - .file_uri = path, - .name = "value", - .shape = .init(.{contents.len}, .u8), - .offset = 0, - }); - try registry.registerTensor(.{ - .file_uri = path, - .name = "second", - .shape = .init(.{second_contents.len}, .u8), - .offset = contents.len, - }); - try registry.registerTensor(.{ - .file_uri = missing_path, - .name = "missing", - .shape = .init(.{contents.len}, .u8), - .offset = 0, - }); - try registry.registerTensor(.{ - .file_uri = path, - .name = "empty", - .shape = .init(.{0}, .u8), - .offset = 0, - }); - self.platform = testing.env(); - self.store = .init(allocator, ®istry, .{ .meshes = &.{self.platform.replicated_mesh} }); - self.value = self.store.view().createReplicatedTensor("value", null); - self.second = self.store.view().createReplicatedTensor("second", null); - self.missing = self.store.view().createReplicatedTensor("missing", null); - self.empty = self.store.view().createReplicatedTensor("empty", null); - } - - fn deinit(self: *LoaderTestFixture) void { - self.store.deinit(); - self.tmp.cleanup(); - } - - /// A tensor of `like`'s shape that `program` computes from the - /// checkpoint tensor `name`. - fn computed(self: *LoaderTestFixture, name: []const u8, like: Tensor, program: anytype) Tensor { - const input = self.store.view().source(name, null).withPartitioning(.replicated, .{}); - return TensorStore.Source.bindProgram(&.{input}, like.shape(), program); - } - - /// The program returning its one input. - const Identity = struct { - pub fn forward(_: Identity, _: Shape, inputs: []const Tensor) Tensor { - return inputs[0]; - } - }; - - /// The program adding `value` to every byte of its one input. - const Add = struct { - value: u8, - - pub fn forward(self: Add, _: Shape, inputs: []const Tensor) Tensor { - return inputs[0].add(Tensor.scalar(self.value, .u8)); - } - }; - - /// Whether `loader` failed with `expected` and holds nothing started. - fn expectFailed(loader: *const Loader, expected: anyerror) !void { - try std.testing.expectEqual(expected, loader.failure.?); - try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); - } -}; - -test "start publishes the bulk at once and wait runs program groups one at a time without memory measurement" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - // One program tensor per group. - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); - defer loader.deinit(); - // Exercise serial admission regardless of the platform's memory stats: - // decided once the set-up is done. - try loader.awaitSetup(); - loader.measured = null; - - const Model = struct { value: Tensor, first: Tensor, second: Tensor }; - const model: Model = .{ - .value = fixture.value, - .first = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), - .second = fixture.computed("second", fixture.second, LoaderTestFixture.Identity{}), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - - try loader.start(&model, &buffers, &fixture.store, .none); - errdefer loader.cancel(); - // The bulk is published; the groups wait for `wait`, their one program - // (the same function and shapes) compiling meanwhile. - try std.testing.expectEqual(1, loader.pending.len); - try std.testing.expectEqual(2, loader.queued.len); - try std.testing.expectEqual(1, loader.programs.items.len); - try loader.wait(); - try std.testing.expectEqual(0, loader.pending.len + loader.queued.len + loader.programs.items.len); - // Serial admission retired the bulk and the first group before it - // published the second. - try std.testing.expectEqual(2, loader.stats.admission_retires); - try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); - try testing.expectShards(allocator, io, &buffers.first, &LoaderTestFixture.contents, .default); - try testing.expectShards(allocator, io, &buffers.second, &LoaderTestFixture.second_contents, .default); - try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); - - // Idempotent: a second wait neither reruns nor recounts. - try loader.wait(); - try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); - - // The next load counts and times from its own first start. - try std.testing.expectEqual(null, loader.stats.started_at); - var again: Buffer = undefined; - try loader.start(&fixture.second, &again, &fixture.store, .none); - try loader.wait(); - defer again.deinit(); - try std.testing.expectEqual(LoaderTestFixture.second_contents.len, loader.stats.bytes_loaded); - try std.testing.expectEqual(null, loader.stats.started_at); -} - -test "one loader takes loads from different stores before one wait" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var other_store = try testing.storeOf(allocator, &fixture.store.registry, .{ .meshes = &.{fixture.platform.replicated_mesh} }); - defer other_store.deinit(); - const other = other_store.view().createReplicatedTensor("second", null); - - var first: Buffer = undefined; - var second: Buffer = undefined; - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - try loader.start(&fixture.value, &first, &fixture.store, .none); - try loader.start(&other, &second, &other_store, .none); - try loader.wait(); - defer first.deinit(); - defer second.deinit(); - try testing.expectShards(allocator, io, &first, &LoaderTestFixture.contents, .default); - try testing.expectShards(allocator, io, &second, &LoaderTestFixture.second_contents, .default); -} - -test "tensors of one program and equal args share one compile, and one group runs them all" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - - const Add = LoaderTestFixture.Add; - const Model = struct { a: Tensor, b: Tensor, c: Tensor }; - const model: Model = .{ - .a = fixture.computed("value", fixture.value, Add{ .value = 1 }), - .b = fixture.computed("second", fixture.second, Add{ .value = 1 }), - .c = fixture.computed("value", fixture.value, Add{ .value = 2 }), - }; - try std.testing.expect(fixture.store.bindingOf(model.a.id).? == .compute); - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - errdefer loader.cancel(); - // Nothing for the loader to fill itself, and one group. - try std.testing.expectEqual(0, loader.pending.len); - try std.testing.expectEqual(1, loader.queued.len); - try std.testing.expectEqual(2, loader.programs.items.len); - try loader.wait(); - try testing.expectShards(allocator, io, &buffers.a, &.{ 2, 3, 4, 5 }, .default); - try testing.expectShards(allocator, io, &buffers.b, &.{ 6, 7, 8, 9 }, .default); - try testing.expectShards(allocator, io, &buffers.c, &.{ 3, 4, 5, 6 }, .default); - try std.testing.expectEqual(LoaderTestFixture.contents.len * 3, loader.stats.bytes_loaded); -} - -test "loader delivers host bindings into pinned host memory without charging the devices" { - for ([_]Memory{ .host_pinned, .host_mapped }) |kind| try expectHostDelivery(kind); -} - -fn expectHostDelivery(kind: Memory) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - const platform = fixture.platform; - const pinned = fixture.store.view().createReplicatedTensor("second", null); - fixture.store.view().setMemory(pinned, kind); - try std.testing.expectEqual(kind, fixture.store.bindingOf(pinned.id).?.load.memory); - - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - const Model = struct { device: Tensor, host: Tensor }; - const model: Model = .{ .device = fixture.value, .host = pinned }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - - if (!backend.supportsMemory(platform, kind)) { - // Refused before anything is submitted. - try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); - try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); - return; - } - try loader.start(&model, &buffers, &fixture.store, .none); - try loader.wait(); - try testing.expectShards(allocator, io, &buffers.device, &LoaderTestFixture.contents, .default); - // One shard per device, each in that device's pinned host memory space - // or, `host_mapped`, a view of pinned memory in its default space. - try testing.expectShards(allocator, io, &buffers.host, &LoaderTestFixture.second_contents, kind); - - // Only the tensor in device memory counts against the devices, on both - // sides of the admission ledger. - if (loader.backend.? == .direct) { - const direct = loader.backend.?.direct; - for (direct.sealed_bytes[0..platform.devices.len], direct.allocated_bytes[0..platform.devices.len]) |sealed, *allocated| { - try std.testing.expectEqual(LoaderTestFixture.contents.len, sealed); - try std.testing.expectEqual(sealed, allocated.load(.acquire)); - } - } -} - -test "a program frees its donated and non-donated inputs once" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); - const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); - defer allocator.free(path); - - const shape: Shape = .init(.{2}, .i32); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); - try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); - defer store.deinit(); - - const Sum = struct { - donate: bool, - - pub fn forward(self: @This(), _: Shape, inputs: []const Tensor) Tensor { - const sum = inputs[0].add(inputs[1]); - return if (self.donate) sum.reuseBuffer(inputs[0]) else sum; - } - }; - for ([_]bool{ false, true }) |donate| { - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - const tensor = TensorStore.Source.bindProgram(&.{ - store.view().source("value", null).withPartitioning(.replicated, .{}), - store.view().source("weight", null).withPartitioning(.replicated, .{}), - }, shape, Sum{ .donate = donate }); - var output: Buffer = undefined; - try loader.start(&tensor, &output, &store, .none); - errdefer loader.cancel(); - try loader.programs.items[0].await(io, platform); - try std.testing.expectEqual(@as(usize, @intFromBool(donate)), loader.programs.items[0].exe.?.donated_input_indices.len); - // Retiring runs the program, then frees every input handle once, the - // donated one included. - try loader.wait(); - defer output.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - } -} - -test "cancel gives up on everything started without running its programs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - - const Model = struct { value: Tensor, computed: Tensor }; - const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - loader.cancel(); - try LoaderTestFixture.expectFailed(&loader, error.Canceled); - try std.testing.expectEqual(@as(usize, 0), buffers.computed._shards.len); - // The loader is failed: what it waits for and what it is given. - try std.testing.expectError(error.Canceled, loader.wait()); - try std.testing.expectError(error.Canceled, loader.start(&model, &buffers, &fixture.store, .none)); - try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); -} - -test "cancel frees a published group's inputs without running its program" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - - const Model = struct { computed: Tensor }; - const model: Model = .{ .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - errdefer loader.cancel(); - // Publish the group as `wait` does, then give up before it retires. - try loader.publish(loader.queued.popFront().?); - try std.testing.expect(loader.programGroupPending()); - loader.cancel(); - try LoaderTestFixture.expectFailed(&loader, error.Canceled); - try std.testing.expect(!loader.programGroupPending()); - try std.testing.expectEqual(0, buffers.computed._shards.len); -} - -test "a read failure fails the loader for every later call" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - // The broken group first, retired when the next one is admitted. - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2, .program_group_bytes = 1 }); - defer loader.deinit(); - try loader.awaitSetup(); - loader.measured = null; - - const Model = struct { broken: Tensor, value: Tensor }; - const model: Model = .{ - .broken = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), - .value = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - try std.testing.expectError(error.FileNotFound, loader.wait()); - try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); - try std.testing.expectEqual(@as(usize, 0), buffers.value._shards.len); - try std.testing.expectError(error.FileNotFound, loader.start(&model, &buffers, &fixture.store, .none)); - try std.testing.expectError(error.FileNotFound, loader.wait()); - try std.testing.expectEqual(@as(usize, 0), loader.stats.bytes_loaded); -} - -test "after a failed load every buffer is empty or whole, and deinitBufferized frees the tree" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - - // A tree with a slice and optionals, some of it in the file that loads, - // a tensor in the file that does not exist, and a program over it. - const Layer = struct { weight: Tensor, bias: ?Tensor }; - const Model = struct { layers: []Layer, broken: Tensor, computed: Tensor }; - var layers = [_]Layer{ - .{ .weight = fixture.value, .bias = fixture.second }, - .{ .weight = fixture.second, .bias = null }, - }; - const model: Model = .{ - .layers = &layers, - .broken = fixture.missing, - .computed = fixture.computed("missing", fixture.missing, LoaderTestFixture.Identity{}), - }; - var buffers = try mem.bufferize(allocator, &model); - // Frees whatever the failed load wrote, under the testing allocator. - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - try std.testing.expectError(error.FileNotFound, loader.wait()); - try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); - // Each buffer holds no shard or one per device. - const devices = fixture.platform.placedShape(fixture.value.shape())._sharding.mesh.?.devicesInCanonicalOrder().len; - var torn: usize = 0; - meta.forEachVisit(&buffers, *Buffer, struct { - fn call(_: usize, buffer: *Buffer, shards: usize, count: *usize) void { - if (buffer._shards.len != 0 and buffer._shards.len != shards) count.* += 1; - } - }.call, .{ devices, &torn }); - try std.testing.expectEqual(0, torn); - try std.testing.expectEqual(0, buffers.computed._shards.len); -} - -test "the direct backend allocates what it was submitted once everything landed" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var placed: DeviceBytes = @splat(0); - // An empty tensor takes no bytes. - addPlacementBytes(&placed, fixture.platform, .init(.{ 0, 4 }, .f32)); - addPlacementBytes(&placed, fixture.platform, fixture.value.shape()); - for (placed[0..fixture.platform.devices.len]) |bytes| try std.testing.expectEqual(LoaderTestFixture.contents.len, bytes); - - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - const Model = struct { value: Tensor, computed: Tensor }; - const model: Model = .{ .value = fixture.value, .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Identity{}) }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - try loader.wait(); - if (loader.backend.? != .direct) return; - const direct = loader.backend.?.direct; - const n = fixture.platform.devices.len; - const unallocated = direct.unallocatedBytes(); - // Replicated program inputs and bulk outputs occupy every device. - for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n], unallocated[0..n]) |sealed, *allocated, bytes| { - try std.testing.expectEqual(LoaderTestFixture.contents.len * 2, sealed); - try std.testing.expectEqual(sealed, allocated.load(.acquire)); - try std.testing.expectEqual(0, bytes); - } -} - -test "init checks every option and returns InvalidOptions" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - const Case = struct { io: std.Io, opts: Loader.Options }; - const cases = [_]Case{ - .{ .io = io, .opts = .{ .read_parallelism = 0 } }, - .{ .io = io, .opts = .{ .read_parallelism = Loader.max_read_parallelism + 1 } }, - .{ .io = io, .opts = .{ .dma_block_size = 0 } }, - .{ .io = io, .opts = .{ .dma_block_size = backend.max_read_request_size + 1 } }, - // No backend of the VFS serves the scheme. - .{ .io = vfs.io(), .opts = .{ .checkpoint = "missing://bucket/model" } }, - }; - for (cases) |case| { - if (Loader.init(allocator, case.io, platform, case.opts)) |value| { - var loader = value; - loader.deinit(); - return error.TestUnexpectedResult; - } else |err| try std.testing.expectEqual(error.InvalidOptions, err); - } - // Under a VFS, a local file may be read directly: a block is a whole - // number of alignment units, reads landing in blocks. - if (VFS.direct_io_alignment != 0) { - if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "/model.safetensors", .dma_block_size = (8 << 20) + VFS.direct_io_alignment / 2 })) |value| { - var loader = value; - loader.deinit(); - return error.TestUnexpectedResult; - } else |err| try std.testing.expectEqual(error.InvalidOptions, err); - // Whatever the checkpoint: a request of one alignment unit cannot - // hold a direct read widened at each end. - try vfs.registerBackend("tiny", .{ .io = io, .read_hints = .{ .read_chunk_size = VFS.direct_io_alignment } }); - if (Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = "tiny://model", .dma_block_size = VFS.direct_io_alignment })) |value| { - var loader = value; - loader.deinit(); - return error.TestUnexpectedResult; - } else |err| try std.testing.expectEqual(error.InvalidOptions, err); - } - // A plain io has no VFS to ask: local sizing. Nothing started, so - // `deinit` cancels the set-up or destroys what it built. - var loader = try Loader.init(allocator, io, platform, .{ .checkpoint = "missing://bucket/model" }); - loader.deinit(); -} - -test "init copies a local checkpoint's path for its set-up" { - if (VFS.direct_io_alignment == 0) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const unit: [VFS.direct_io_alignment]u8 = @splat(7); - try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = &unit }); - var root_buffer: [std.fs.max_path_bytes]u8 = undefined; - const root = root_buffer[0..try tmp.dir.realPath(io, &root_buffer)]; - var path_buffer: [std.fs.max_path_bytes + 32]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buffer, "{s}/model.safetensors", .{root}); - - // The copy is the only allocation of `init`: nothing is left to free - // when it fails. - var failing: std.testing.FailingAllocator = .init(allocator, .{ .fail_index = 0 }); - if (Loader.init(failing.allocator(), vfs.io(), platform, .{ .checkpoint = path })) |value| { - var loader = value; - loader.deinit(); - return error.TestUnexpectedResult; - } else |err| try std.testing.expectEqual(error.OutOfMemory, err); - - // The set-up frees `init`'s copy once, whether it ran to its end or - // `deinit` canceled it before any `start`. - var loader = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path }); - defer loader.deinit(); - try loader.awaitSetup(); - for ([_]VFS.DirectIo{ .on, .auto }) |policy| { - var unstarted = try Loader.init(allocator, vfs.io(), platform, .{ .checkpoint = path, .direct_io = policy }); - unstarted.deinit(); - } -} - -test "a failed set-up fails the first start, and deinit frees the rest" { - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(std.testing.allocator, io); - defer fixture.deinit(); - // `init` allocates nothing; the set-up's third allocation, inside the - // backend, fails, so the backend unwinds what it built before it. - var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 2 }); - var loader = try Loader.init(failing.allocator(), io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - var output: Buffer = undefined; - try std.testing.expectError(error.OutOfMemory, loader.start(&fixture.value, &output, &fixture.store, .none)); - try LoaderTestFixture.expectFailed(&loader, error.OutOfMemory); - try std.testing.expect(loader.backend == null); - try std.testing.expectError(error.OutOfMemory, loader.wait()); -} - -test "without concurrency init sets up in place" { - const allocator = std.testing.allocator; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, std.testing.io); - defer fixture.deinit(); - // The testing io, but `io.concurrent` refuses and `io.async` runs the - // function in place. - var vtable = std.testing.io.vtable.*; - vtable.concurrent = struct { - fn concurrent(_: ?*anyopaque, _: usize, _: std.mem.Alignment, _: []const u8, _: std.mem.Alignment, _: *const fn (*const anyopaque, *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { - return error.ConcurrencyUnavailable; - } - }.concurrent; - vtable.async = struct { - fn async(_: ?*anyopaque, result: []u8, _: std.mem.Alignment, context: []const u8, _: std.mem.Alignment, start: *const fn (*const anyopaque, *anyopaque) void) ?*std.Io.AnyFuture { - start(context.ptr, result.ptr); - return null; - } - }.async; - const io: std.Io = .{ .userdata = std.testing.io.userdata, .vtable = &vtable }; - - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - try std.testing.expect(loader.setup.?.any_future == null); - // Nothing started: neither has anything to do. - loader.cancel(); - try loader.wait(); - try std.testing.expectEqual(null, loader.failure); - - var output: Buffer = undefined; - try loader.start(&fixture.value, &output, &fixture.store, .none); - try loader.wait(); - defer output.deinit(); - try testing.expectShards(allocator, io, &output, &LoaderTestFixture.contents, .default); -} - -test "a rate-limited HTTP source loads through the VFS hold" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const tensor_bytes = 1024; - const tensors = 4; - - // Four tensor ranges with a gap between them, so the planner keeps four - // jobs and the load makes four concurrent GETs. - var object: [(2 * tensors - 1) * tensor_bytes]u8 = undefined; - for (&object, 0..) |*byte, index| byte.* = @truncate(index *% 31 +% 7); - - var server = try VFS.MockServer.init(io, &object, .{ - // Two GETs per 100 ms: the four readers are throttled repeatedly. - .throttle = .{ .window = .{ .gets = 2, .per_ms = 100 } }, - }); - defer server.deinit(io); - try server.start(io); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - var http: VFS.HTTP = try .init(allocator, io, &client, .{ .protocol = .http, .retry = .{ - .max_retries = 5, - .initial_delay = .fromMilliseconds(20), - .max_delay = .fromMilliseconds(20), - .max_hold = .fromSeconds(5), - .throttle_budget = .fromSeconds(30), - } }); - defer http.deinit(); - var vfs: VFS = try .init(allocator, io); - defer vfs.deinit(); - try vfs.registerBackend("http", http.backend()); - const stats = &http.governor.stats; - - var url_buffer: [128]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, "http://127.0.0.1:{d}/object", .{server.port()}); - - const platform = testing.env(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - const names = [tensors][]const u8{ "a", "b", "c", "d" }; - for (names, 0..) |name, index| { - try registry.registerTensor(.{ - .file_uri = url, - .name = name, - .shape = .init(.{tensor_bytes}, .u8), - .offset = 2 * index * tensor_bytes, - }); - } - var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{platform.replicated_mesh} }); - defer store.deinit(); - - const Model = struct { a: Tensor, b: Tensor, c: Tensor, d: Tensor }; - const model: Model = .{ - .a = store.view().createReplicatedTensor("a", null), - .b = store.view().createReplicatedTensor("b", null), - .c = store.view().createReplicatedTensor("c", null), - .d = store.view().createReplicatedTensor("d", null), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - - var loader = try Loader.init(allocator, vfs.io(), platform, .{ - .checkpoint = url, - .read_parallelism = 4, - }); - // Holding costs time, not credits: the direct backend's `destroy` - // asserts that every lifecycle credit came back with its pinned blocks. - // It checks once its workers stopped, since until then one may hold a - // credit for a claim that finds nothing. - defer loader.deinit(); - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - - inline for (comptime std.meta.fieldNames(Model), 0..) |field_name, index| { - try testing.expectShards(allocator, io, &@field(buffers, field_name), object[2 * index * tensor_bytes ..][0..tensor_bytes], .default); - } - - try server.stop(io); - - // The server rate limited the load, the backend held every reader, and - // no reader burned its retry budget for it. - try std.testing.expect(stats.throttles.load(.acquire) > 0); - try std.testing.expect(stats.holds.load(.acquire) > 0); - try std.testing.expect(stats.hold_wait_ns.load(.acquire) > 0); - try std.testing.expectEqual(@as(u64, 0), stats.retries.load(.acquire)); - try std.testing.expectEqual(@as(u64, tensors * tensor_bytes), stats.physical_bytes.load(.acquire)); - - // The checkpoint's VFS backend, not the local hints (8 MiB), sized the - // reads: 16 MiB requests over the CPU's 2 MiB blocks. Its file was read - // exact: a remote file is never direct. - const direct = loader.backend.?.direct; - try std.testing.expectEqual(@as(usize, 16 << 20), direct.plan_config.request_size); - try std.testing.expectEqual(@as(usize, 0), direct.source_slots.get(url).?.alignment orelse 0); -} - -test "a rejected binding or a program of the wrong output fails the loader" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - const platform = fixture.platform; - // Two rows of two, not the four bytes the tensor holds. - const Rows = struct { - pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { - return inputs[0].reshape(.{ 2, 2 }); - } - }; - // A mesh of devices the platform does not have: the same tensors placed - // on it, in device and host memory, and a program's output. - var foreign_physical = platform.replicated_mesh.physical.*; - const foreign_devices = try allocator.dupe(Sharding.Device, foreign_physical.devices_in_canonical_order); - defer allocator.free(foreign_devices); - for (foreign_devices) |*device| device.id += 100; - foreign_physical.devices_in_canonical_order = foreign_devices; - var foreign_mesh = platform.replicated_mesh.*; - foreign_mesh.physical = &foreign_physical; - const pinned = fixture.store.view().createReplicatedTensor("second", null); - fixture.store.view().setMemory(pinned, .host_pinned); - var foreign = [_]Tensor{ fixture.value, pinned, fixture.value }; - for (&foreign) |*tensor| tensor._shape._sharding.mesh = &foreign_mesh; - // The program places its output replicated, the tensor expects its - // bytes split over the devices: same dims, another placement. It needs - // devices that split four bytes. - var misplaced = fixture.value; - misplaced._shape = Shape.init(.{ .x = 4 }, .u8).withPartitioning(platform.meshes.get("model").?, .{ .x = .model }); - const splits = platform.devices.len > 1 and 4 % platform.devices.len == 0; - - // No binding, an empty source or program input, a mesh of other - // devices, and a program of the wrong shape or placement. - const Case = struct { tensor: Tensor, at_wait: bool = false, skip: bool = false }; - const cases = [_]Case{ - .{ .tensor = Tensor.fromShape(fixture.value.shape()) }, - .{ .tensor = fixture.empty }, - .{ .tensor = fixture.computed("empty", fixture.empty, LoaderTestFixture.Identity{}) }, - .{ .tensor = foreign[0] }, - .{ .tensor = foreign[1] }, - .{ .tensor = fixture.computed("value", foreign[2], LoaderTestFixture.Identity{}) }, - .{ .tensor = fixture.computed("value", fixture.value, Rows{}), .at_wait = true }, - .{ .tensor = fixture.computed("value", misplaced, LoaderTestFixture.Identity{}), .at_wait = true, .skip = !splits }, - }; - for (cases) |case| { - if (case.skip) continue; - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - var output: Buffer = undefined; - if (case.at_wait) { - try loader.start(&case.tensor, &output, &fixture.store, .none); - try std.testing.expectError(error.InvalidBinding, loader.wait()); - } else { - try std.testing.expectError(error.InvalidBinding, loader.start(&case.tensor, &output, &fixture.store, .none)); - } - try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); - // `start` emptied the buffer before it failed. - try std.testing.expectEqual(0, output._shards.len); - output.deinit(); - try std.testing.expectError(error.InvalidBinding, loader.start(&fixture.value, &output, &fixture.store, .none)); - } -} - -test "buffers of another structure fail start before any buffer is written" { - var fixture: LoaderTestFixture = undefined; - try fixture.init(std.testing.allocator, std.testing.io); - defer fixture.deinit(); - const value = fixture.value; - const second = fixture.second; - - // Each pair holds as many tensors on both sides, in another structure, - // and a pair precedes each mismatch. - const Optionals = struct { head: Tensor, a: ?Tensor, b: ?Tensor }; - try expectStructureMismatch(&fixture, Optionals, .{ .head = value, .a = value, .b = null }, .{ .head = value, .a = null, .b = value }); - const Arms = struct { head: Tensor, arm: union(enum) { a: Tensor, b: Tensor }, last: Tensor }; - try expectStructureMismatch(&fixture, Arms, .{ .head = value, .arm = .{ .a = value }, .last = second }, .{ .head = value, .arm = .{ .b = value }, .last = second }); - const Slices = struct { head: Tensor, first: []Tensor, second: []Tensor }; - var two = [_]Tensor{ value, second }; - var one = [_]Tensor{value}; - var other_one = [_]Tensor{second}; - var other_two = [_]Tensor{ second, value }; - try expectStructureMismatch(&fixture, Slices, .{ .head = value, .first = &two, .second = &one }, .{ .head = value, .first = &other_one, .second = &other_two }); -} - -/// Starts `model` into the buffers bufferized from `built`, which has as -/// many tensors in another structure: `InvalidBinding`, and every buffer as -/// it was. -fn expectStructureMismatch(fixture: *LoaderTestFixture, comptime Model: type, built: Model, model: Model) !void { - const allocator = std.testing.allocator; - var loader = try Loader.init(allocator, std.testing.io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - var buffers = try mem.bufferize(allocator, &built); - defer mem.deinitBufferized(allocator, &buffers); - // No tensor has this shape, which `start` would replace with its - // tensor's when it empties a buffer. - const untouched: Shape = .init(.{ 1, 2, 3 }, .u8); - meta.forEachVisit(&buffers, *Buffer, struct { - fn call(_: usize, buffer: *Buffer, shape: Shape) void { - buffer._shape = shape; - } - }.call, .{untouched}); - try std.testing.expectError(error.InvalidBinding, loader.start(&model, &buffers, &fixture.store, .none)); - try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); - var written: usize = 0; - meta.forEachVisit(&buffers, *Buffer, struct { - fn call(_: usize, buffer: *Buffer, shape: Shape, count: *usize) void { - if (!buffer._shape.eql(shape) or buffer._shards.len != 0) count.* += 1; - } - }.call, .{ untouched, &written }); - try std.testing.expectEqual(0, written); -} - -/// Two safetensors files for composite loads, every row 16 bytes (8 u16): -/// `first` holds `a` [3, 8] then `b` [2, 8], adjacent, then `spacer`, then -/// `q` [8, 8], `k` [4, 8] and `v` [4, 8]; `second` holds `plain` [4, 8], -/// then `c` [4, 8], the experts `e{0..3}.w1` and `e{0..3}.w3` [2, 8] and -/// `wide` [64, 8]. `empty` [0, 8] is registered without bytes. Pinned after -/// `init`. Each store takes a copy of `registry` (`testing.storeOf`). -const CompositeFixture = struct { - tmp: std.testing.TmpDir, - registry: safetensors.TensorRegistry, - a: [48]u8, - b: [32]u8, - c: [64]u8, - plain: [64]u8, - q: [128]u8, - k: [64]u8, - v: [64]u8, - w1: [4][32]u8, - w3: [4][32]u8, - wide: [1024]u8, - - const expert_names = [2][4][]const u8{ - .{ "e0.w1", "e1.w1", "e2.w1", "e3.w1" }, - .{ "e0.w3", "e1.w3", "e2.w3", "e3.w3" }, - }; - - fn init(self: *CompositeFixture, allocator: std.mem.Allocator, io: std.Io) !void { - self.tmp = std.testing.tmpDir(.{}); - errdefer self.tmp.cleanup(); - self.registry = .init(allocator); - errdefer self.registry.deinit(); - testing.pattern(&self.a, 1); - testing.pattern(&self.b, 2); - testing.pattern(&self.c, 3); - testing.pattern(&self.plain, 4); - testing.pattern(&self.q, 9); - testing.pattern(&self.k, 10); - testing.pattern(&self.v, 11); - for (&self.w1, &self.w3, 0..) |*w1, *w3, e| { - testing.pattern(w1, @intCast(20 + e)); - testing.pattern(w3, @intCast(30 + e)); - } - var spacer: [16]u8 = undefined; - testing.pattern(&spacer, 5); - testing.pattern(&self.wide, 6); - try testing.writeSafetensors(allocator, io, self.tmp.dir, "first.safetensors", &self.registry, &.{ - .{ .name = "a", .dtype = "U16", .dims = &.{ 3, 8 }, .bytes = &self.a }, - .{ .name = "b", .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.b }, - .{ .name = "spacer", .dtype = "U16", .dims = &.{ 1, 8 }, .bytes = &spacer }, - .{ .name = "q", .dtype = "U16", .dims = &.{ 8, 8 }, .bytes = &self.q }, - .{ .name = "k", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.k }, - .{ .name = "v", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.v }, - }); - try testing.writeSafetensors(allocator, io, self.tmp.dir, "second.safetensors", &self.registry, &.{ - .{ .name = "plain", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.plain }, - .{ .name = "c", .dtype = "U16", .dims = &.{ 4, 8 }, .bytes = &self.c }, - .{ .name = expert_names[0][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[0] }, - .{ .name = expert_names[1][0], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[0] }, - .{ .name = expert_names[0][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[1] }, - .{ .name = expert_names[1][1], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[1] }, - .{ .name = expert_names[0][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[2] }, - .{ .name = expert_names[1][2], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[2] }, - .{ .name = expert_names[0][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w1[3] }, - .{ .name = expert_names[1][3], .dtype = "U16", .dims = &.{ 2, 8 }, .bytes = &self.w3[3] }, - .{ .name = "wide", .dtype = "U16", .dims = &.{ 64, 8 }, .bytes = &self.wide }, - }); - try self.registry.registerTensor(.{ - .file_uri = self.registry.tensors.get("a").?.file_uri, - .name = "empty", - .shape = .init(.{ 0, 8 }, .u16), - .offset = 0, - }); - } - - fn deinit(self: *CompositeFixture) void { - self.registry.deinit(); - self.tmp.cleanup(); - } - - fn rows(view: TensorStore.View, name: []const u8) TensorStore.Source { - return view.source(name, .{ .row, .col }); - } -}; - -test "loader assembles concatenated and padded sources on every placement and memory kind" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: CompositeFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - // a, b and c end to end over 16 rows; c alone over 8 rows; b then a - // over 6 rows. The rest is zero. - var concatenated: [256]u8 = @splat(0); - @memcpy(concatenated[0..48], &fixture.a); - @memcpy(concatenated[48..80], &fixture.b); - @memcpy(concatenated[80..144], &fixture.c); - var padded: [128]u8 = @splat(0); - @memcpy(padded[0..64], &fixture.c); - var reversed: [96]u8 = @splat(0); - @memcpy(reversed[0..32], &fixture.b); - @memcpy(reversed[32..80], &fixture.a); - - const Source = TensorStore.Source; - const rows = CompositeFixture.rows; - const Model = struct { concatenated: Tensor, padded: Tensor, reversed: Tensor, plain: Tensor }; - for ([_]usize{ 2, 4 }) |device_count| { - var devices: testing.Devices = undefined; - try devices.init(allocator, io, device_count); - defer devices.deinit(); - const platform = devices.platform; - const tp = try devices.mesh("tp"); - var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); - defer store.deinit(); - // Rows over the devices (b and c straddle shards on four), every - // row split over them, and every byte on each of them. - inline for (.{ .{ .row = .model }, .{ .col = .model }, .{} }) |partitioning| { - const shards: u64 = if (@typeInfo(@TypeOf(partitioning)).@"struct".field_names.len == 0) 1 else device_count; - for ([_]Memory{ .default, .host_pinned, .host_mapped }) |memory| { - if (!backend.supportsMemory(platform, memory)) continue; - const view = store.view(); - const model: Model = .{ - .concatenated = Source.concatenate(&.{ rows(view, "a"), rows(view, "b"), rows(view, "c") }, .row) - .pad(0, .{ .row = Source.Pad{ .high = 7 } }) - .withPartitioning(.tp, partitioning).bind(), - // The padded engram-style table: one source. - .padded = rows(view, "c").pad(0, .{ .row = Source.Pad{ .high = 4 } }).withPartitioning(.tp, partitioning).bind(), - // Out of file order, on the platform's replicated mesh. - .reversed = Source.concatenate(&.{ rows(view, "b"), rows(view, "a") }, .row) - .pad(0, .{ .row = Source.Pad{ .high = 1 } }) - .withPartitioning(.replicated, .{}).bind(), - .plain = rows(view, "plain").withPartitioning(.tp, partitioning).bind(), - }; - inline for (comptime std.meta.fieldNames(Model)) |name| view.setMemory(@field(model, name), memory); - try std.testing.expect(store.bindingOf(model.plain.id).?.load.source.whole() != null); - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - - try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated, memory); - try testing.expectShards(allocator, io, &buffers.padded, &padded, memory); - try testing.expectShards(allocator, io, &buffers.reversed, &reversed, memory); - try testing.expectShards(allocator, io, &buffers.plain, &fixture.plain, memory); - // Each destination once, padding included. - try std.testing.expectEqual(@as(usize, 256 + 128 + 96 + 64), loader.stats.bytes_loaded); - // A destination is allocated once per device, whatever its - // parts; host memory counts on neither side. - if (loader.backend.? == .direct) { - const direct = loader.backend.?.direct; - const n = platform.devices.len; - for (direct.sealed_bytes[0..n], direct.allocated_bytes[0..n]) |sealed, *allocated| try std.testing.expectEqual(sealed, allocated.load(.acquire)); - for (devices.physical.devices_in_canonical_order) |device| { - const expected: u64 = if (memory.isHost()) 0 else (256 + 128 + 64) / shards + 96; - try std.testing.expectEqual(expected, direct.sealed_bytes[device.id]); - } - } - } - } - } -} - -test "loader assembles interleaved, stacked and split sources" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: CompositeFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - const Source = TensorStore.Source; - const rows = CompositeFixture.rows; - const row = 16; - - // Experts with their two projections concatenated, then row by row. - var concatenated_experts: [256]u8 = undefined; - var interleaved_experts: [256]u8 = undefined; - for (0..4) |e| { - @memcpy(concatenated_experts[e * 64 ..][0..32], &fixture.w1[e]); - @memcpy(concatenated_experts[e * 64 + 32 ..][0..32], &fixture.w3[e]); - for (0..2) |r| { - @memcpy(interleaved_experts[e * 64 + r * 2 * row ..][0..row], fixture.w1[e][r * row ..][0..row]); - @memcpy(interleaved_experts[e * 64 + (r * 2 + 1) * row ..][0..row], fixture.w3[e][r * row ..][0..row]); - } - } - - const Model = struct { qkv: Tensor, concatenated: Tensor, interleaved: Tensor, head: Tensor, tail: Tensor }; - for ([_]usize{ 2, 4 }) |device_count| { - var devices: testing.Devices = undefined; - try devices.init(allocator, io, device_count); - defer devices.deinit(); - const platform = devices.platform; - const tp = try devices.mesh("tp"); - var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{&tp} }); - defer store.deinit(); - const view = store.view(); - const groups: i64 = @intCast(device_count); - - // q, k and v cut in one group per device: each device holds its rows - // of every member, contiguous. Every part has `groups` runs. - var members: [3]Source = undefined; - for (&members, [_][]const u8{ "q", "k", "v" }) |*member, name| { - member.* = rows(view, name).splitAxis(.row, .{ .group = groups, .r = -1 }); - } - var experts: [2]Source = undefined; - for (&experts, CompositeFixture.expert_names) |*stacked, names| { - var per_expert: [4]Source = undefined; - for (&per_expert, names) |*expert, name| expert.* = rows(view, name); - stacked.* = Source.stack(&per_expert, 0, .expert); - } - const fused = Source.concatenate(&.{ rows(view, "q"), rows(view, "k") }, .row); - // A part of q, and the rest of q with all of k. - const head, const tail = fused.split(.row, &.{ 4, 8 })[0..2].*; - const model: Model = .{ - .qkv = Source.concatenate(&members, .r).merge(.{ .row = .{ .group, .r } }).withPartitioning(.tp, .{ .row = .model }).bind(), - // Experts are whole rows of the merged axis, one or two per device. - .concatenated = Source.concatenate(&experts, .row) - .merge(.{ .rows = .{ .expert, .row } }) - .withPartitioning(.tp, .{ .rows = .model }).bind(), - .interleaved = Source.stack(&experts, 2, .proj) - .merge(.{ .rows = .{ .expert, .row, .proj } }) - .withPartitioning(.tp, .{ .rows = .model }).bind(), - .head = head.withPartitioning(.replicated, .{}).bind(), - .tail = tail.withPartitioning(.tp, .{ .row = .model }).bind(), - }; - var lowering: std.heap.ArenaAllocator = .init(allocator); - defer lowering.deinit(); - const qkv_parts = try store.bindingOf(model.qkv.id).?.load.source.lower(lowering.allocator()); - try std.testing.expectEqual(@as(usize, 3), qkv_parts.len); - for (qkv_parts) |part| try std.testing.expectEqual(@as(u64, device_count), part.lattice.runs()); - - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - - var qkv: [256]u8 = undefined; - var at: usize = 0; - for (0..device_count) |g| { - for ([_][]const u8{ &fixture.q, &fixture.k, &fixture.v }) |member| { - const per_group = member.len / device_count; - @memcpy(qkv[at..][0..per_group], member[g * per_group ..][0..per_group]); - at += per_group; - } - } - try testing.expectShards(allocator, io, &buffers.qkv, &qkv, .default); - try testing.expectShards(allocator, io, &buffers.concatenated, &concatenated_experts, .default); - try testing.expectShards(allocator, io, &buffers.interleaved, &interleaved_experts, .default); - try testing.expectShards(allocator, io, &buffers.head, fixture.q[0..64], .default); - var tail_bytes: [128]u8 = undefined; - @memcpy(tail_bytes[0..64], fixture.q[64..128]); - @memcpy(tail_bytes[64..], &fixture.k); - try testing.expectShards(allocator, io, &buffers.tail, &tail_bytes, .default); - } -} - -test "loader gathers strided sources: a row interleave across files, broadcasts and a transpose" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const Source = TensorStore.Source; - // Rows of 16 bytes: g and u interleave row by row, 96 runs each; s0 and - // s1 repeat 100 and 60 times; z [24, 4] is transposed, 96 runs of two - // bytes, then padded with two zero rows. Every part has more runs than - // expand, so the direct backend gathers them, its jobs reading g's rows - // from one file and u's from the other. - var g: [96 * 16]u8 = undefined; - var u: [96 * 16]u8 = undefined; - var z: [24 * 8]u8 = undefined; - var s0: [4]u8 = undefined; - var s1: [4]u8 = undefined; - testing.pattern(&g, 1); - testing.pattern(&u, 2); - testing.pattern(&z, 3); - testing.pattern(&s0, 4); - testing.pattern(&s1, 5); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "first.safetensors", ®istry, &.{ - .{ .name = "g", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &g }, - .{ .name = "s0", .dtype = "F32", .dims = &.{}, .bytes = &s0 }, - .{ .name = "z", .dtype = "U16", .dims = &.{ 24, 4 }, .bytes = &z }, - }); - try testing.writeSafetensors(allocator, io, tmp.dir, "second.safetensors", ®istry, &.{ - .{ .name = "s1", .dtype = "F32", .dims = &.{}, .bytes = &s1 }, - .{ .name = "u", .dtype = "U16", .dims = &.{ 96, 8 }, .bytes = &u }, - }); - - var interleaved: [192 * 16]u8 = undefined; - for (0..96) |r| { - @memcpy(interleaved[2 * r * 16 ..][0..16], g[r * 16 ..][0..16]); - @memcpy(interleaved[(2 * r + 1) * 16 ..][0..16], u[r * 16 ..][0..16]); - } - var scales: [160 * 4]u8 = undefined; - for (0..160) |i| @memcpy(scales[i * 4 ..][0..4], if (i < 100) &s0 else &s1); - var transposed: [6 * 48]u8 = @splat(0); - for (0..4) |column| { - for (0..24) |row| @memcpy(transposed[column * 48 + row * 2 ..][0..2], z[row * 8 + column * 2 ..][0..2]); - } - - const Model = struct { by_rows: Tensor, by_columns: Tensor, scales_by_rows: Tensor, scales_by_columns: Tensor, transposed: Tensor, transposed_by_columns: Tensor }; - for ([_]usize{ 1, 2, 4 }) |device_count| { - for ([_]usize{ 0, 64 }) |alignment| { - var devices: testing.Devices = undefined; - try devices.init(allocator, io, device_count); - defer devices.deinit(); - const tp = try devices.mesh("tp"); - var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); - defer store.deinit(); - const view = store.view(); - const rows = Source.stack(&.{ view.source("g", .{ .row, .col }), view.source("u", .{ .row, .col }) }, 1, .proj) - .merge(.{ .row = .{ .row, .proj } }); - const repeated = Source.concatenate(&.{ - view.source("s0", .{}).broad(Shape.init(.{ .c = 100 }, .f32)), - view.source("s1", .{}).broad(Shape.init(.{ .c = 60 }, .f32)), - }, .c).splitAxis(.c, .{ .row = 40, .col = 4 }); - const flipped = view.source("z", .{ .row, .col }).transpose(.{ .col, .row }).withTags(.{ .row, .col }) - .pad(0, .{ .row = Tensor.Pad{ .high = 2 } }); - const model: Model = .{ - .by_rows = rows.withPartitioning(.tp, .{ .row = .model }).bind(), - .by_columns = rows.withPartitioning(.tp, .{ .col = .model }).bind(), - .scales_by_rows = repeated.withPartitioning(.tp, .{ .row = .model }).bind(), - .scales_by_columns = repeated.withPartitioning(.tp, .{ .col = .model }).bind(), - .transposed = flipped.withPartitioning(.replicated, .{}).bind(), - .transposed_by_columns = flipped.withPartitioning(.tp, .{ .col = .model }).bind(), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - // Jobs of 128 bytes: every destination takes several. - try loader.awaitSetup(); - switch (loader.backend.?) { - .direct => |direct| { - direct.plan_config.request_size = 256; - direct.reads.alignment = alignment; - }, - .buffered => {}, - } - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - try testing.expectShards(allocator, io, &buffers.by_rows, &interleaved, .default); - try testing.expectShards(allocator, io, &buffers.by_columns, &interleaved, .default); - try testing.expectShards(allocator, io, &buffers.scales_by_rows, &scales, .default); - try testing.expectShards(allocator, io, &buffers.scales_by_columns, &scales, .default); - try testing.expectShards(allocator, io, &buffers.transposed, &transposed, .default); - try testing.expectShards(allocator, io, &buffers.transposed_by_columns, &transposed, .default); - } - } -} - -test "loader gathers a block-scale swizzle, its padding zero" { - const allocator = std.testing.allocator; - const io = std.testing.io; - // Scales [e=4, m=31, k=6] in the 128x4 layout at 8x4: m padded to 32 - // and tiled by 2x4, k padded to 8. Its parts repeat runs of four bytes - // or two, more than expand, so the direct backend gathers them. - var scales: [4 * 31 * 6]u8 = undefined; - testing.pattern(&scales, 7); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "scales.safetensors", ®istry, &.{ - .{ .name = "scales", .dtype = "U8", .dims = &.{ 4, 31, 6 }, .bytes = &scales }, - }); - var swizzled: [4 * 32 * 8]u8 = @splat(0); - for (0..4) |e| for (0..4) |mo| for (0..2) |ko| for (0..4) |mi| for (0..2) |m4| for (0..4) |ki| { - const m = mo * 8 + m4 * 4 + mi; - const k = ko * 4 + ki; - const flat = (((mo * 2 + ko) * 4 + mi) * 2 + m4) * 4 + ki; - if (m < 31 and k < 6) swizzled[e * 256 + flat] = scales[(e * 31 + m) * 6 + k]; - }; - - const Model = struct { replicated: Tensor, by_rows: Tensor }; - for ([_]usize{ 1, 2, 4 }) |device_count| { - for ([_]usize{ 0, 64 }) |alignment| { - var devices: testing.Devices = undefined; - try devices.init(allocator, io, device_count); - defer devices.deinit(); - const tp = try devices.mesh("tp"); - var store = try testing.storeOf(allocator, ®istry, .{ .meshes = &.{&tp} }); - defer store.deinit(); - const swizzle = store.view().source("scales", .{ .e, .m, .k }) - .pad(0, .{ .m = Tensor.Pad{ .high = 1 }, .k = Tensor.Pad{ .high = 2 } }) - .splitAxis(.m, .{ .mo = -1, .m4 = 2, .mi = 4 }) - .splitAxis(.k, .{ .ko = -1, .ki = 4 }) - .transpose(.{ .e, .mo, .ko, .mi, .m4, .ki }) - .merge(.{ .flat = .{ .mo, .ko, .mi, .m4, .ki } }); - const model: Model = .{ - .replicated = swizzle.withPartitioning(.replicated, .{}).bind(), - .by_rows = swizzle.withPartitioning(.tp, .{ .e = .model }).bind(), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, devices.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - // Jobs of at most 1024 bytes, each reading the 744 source - // bytes whole: the swizzle reorders its runs. - try loader.awaitSetup(); - switch (loader.backend.?) { - .direct => |direct| { - direct.plan_config.request_size = 1024; - direct.reads.alignment = alignment; - }, - .buffered => {}, - } - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - try testing.expectShards(allocator, io, &buffers.replicated, &swizzled, .default); - try testing.expectShards(allocator, io, &buffers.by_rows, &swizzled, .default); - } - } -} - -test "bindings the loader cannot fill fail it before submission" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: CompositeFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - // Three nibbles: whole bytes hold no element boundary past the first two. - try fixture.registry.registerTensor(.{ - .file_uri = fixture.registry.tensors.get("a").?.file_uri, - .name = "nibbles", - .shape = .init(.{3}, .u4), - .offset = 0, - }); - try fixture.registry.registerTensor(.{ - .file_uri = fixture.registry.tensors.get("a").?.file_uri, - .name = "nibble_rows", - .shape = .init(.{ 2, 3 }, .f4e2m1), - .offset = 0, - }); - const platform = testing.env(); - var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); - defer store.deinit(); - const Source = TensorStore.Source; - const view = store.view(); - const rows = CompositeFixture.rows; - - const rejected = [_]Tensor{ - // Nothing to read: an empty tensor padded. - rows(view, "empty").pad(0, .{ .row = Source.Pad{ .high = 8 } }).withPartitioning(.replicated, .{}).bind(), - // Twelve bits: its one run ends inside a byte (no known checkpoint - // has one). - view.createReplicatedTensor("nibbles", null), - // Strided parts into host memory. - host: { - const tensor = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(); - view.setMemory(tensor, .host_pinned); - break :host tensor; - }, - // Three whole bytes, in rows of three nibbles that do not pack, as a - // tensor and as a program input. - view.createReplicatedTensor("nibble_rows", null), - Source.bindProgram(&.{view.source("nibble_rows", null).withPartitioning(.replicated, .{})}, .init(.{3}, .u8), LoaderTestFixture.Identity{}), - // A program input whose transpose reads its 1024 source bytes in - // every job, more than one read. - Source.bindProgram(&.{rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{})}, .init(.{ 8, 64 }, .u16), LoaderTestFixture.Identity{}), - }; - var output: Buffer = undefined; - for (&rejected) |*tensor| { - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - // One read holds half of `wide`; the buffered backend reads any part. - try loader.awaitSetup(); - switch (loader.backend.?) { - .direct => |direct| direct.plan_config.request_size = 512, - .buffered => if (tensor == &rejected[rejected.len - 1]) continue, - } - try std.testing.expectError(error.InvalidBinding, loader.start(tensor, &output, &store, .none)); - try LoaderTestFixture.expectFailed(&loader, error.InvalidBinding); - } -} - -test "loader fills gaps, reordered and strided rows, and a transpose" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: CompositeFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - const platform = testing.env(); - var store = try testing.storeOf(allocator, &fixture.registry, .{ .meshes = &.{platform.replicated_mesh} }); - defer store.deinit(); - const Source = TensorStore.Source; - const Pad = Source.Pad; - const view = store.view(); - const rows = CompositeFixture.rows; - const a = rows(view, "a"); - - // b's second row past a gap, the rest zero; b then a, each padded; a's - // last row before its first two, one source feeding two parts; a's - // rows every other row, a strided part of few runs, expanded; and a - // transpose, 512 runs of two bytes, gathered. - const Model = struct { sparse: Tensor, padded: Tensor, swapped: Tensor, strided: Tensor, transposed: Tensor }; - const model: Model = .{ - .sparse = rows(view, "b").slice(.row, .{ .start = 1 }).pad(0, .{ .row = Pad{ .low = 4, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), - .padded = Source.concatenate(&.{ - rows(view, "b").pad(0, .{ .row = Pad{ .high = 2 } }), - a.pad(0, .{ .row = Pad{ .high = 1 } }), - }, .row).withPartitioning(.replicated, .{}).bind(), - .swapped = Source.concatenate(&.{ a.slice(.row, .{ .start = 2 }), a.slice(.row, .{ .end = 2 }) }, .row) - .pad(0, .{ .row = Pad{ .high = 5 } }).withPartitioning(.replicated, .{}).bind(), - .strided = a.pad(0, .{ .row = Pad{ .interior = 1, .high = 3 } }).withPartitioning(.replicated, .{}).bind(), - .transposed = rows(view, "wide").transpose(.{}).withPartitioning(.replicated, .{}).bind(), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - try loader.start(&model, &buffers, &store, .none); - try loader.wait(); - - var expected: [128]u8 = @splat(0); - @memcpy(expected[64..80], fixture.b[16..32]); - try testing.expectShards(allocator, io, &buffers.sparse, &expected, .default); - expected = @splat(0); - @memcpy(expected[0..32], &fixture.b); - @memcpy(expected[64..112], &fixture.a); - try testing.expectShards(allocator, io, &buffers.padded, &expected, .default); - expected = @splat(0); - @memcpy(expected[0..16], fixture.a[32..48]); - @memcpy(expected[16..48], fixture.a[0..32]); - try testing.expectShards(allocator, io, &buffers.swapped, &expected, .default); - expected = @splat(0); - for (0..3) |i| @memcpy(expected[i * 32 ..][0..16], fixture.a[i * 16 ..][0..16]); - try testing.expectShards(allocator, io, &buffers.strided, &expected, .default); - var transposed_bytes: [1024]u8 = undefined; - for (0..8) |column| { - for (0..64) |row| @memcpy(transposed_bytes[column * 128 + row * 2 ..][0..2], fixture.wide[row * 16 + column * 2 ..][0..2]); - } - try testing.expectShards(allocator, io, &buffers.transposed, &transposed_bytes, .default); -} - -test "a program takes its inputs from sources, in order" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - const Source = TensorStore.Source; - const view = fixture.store.view(); - - // value's first half then second's last half, assembled by the loader, - // and second as it is: the program subtracts the second input from the - // first, which tells them apart. - const Sub = struct { - pub fn forward(_: @This(), _: Shape, inputs: []const Tensor) Tensor { - return inputs[0].sub(inputs[1]); - } - }; - const assembled = Source.concatenate(&.{ - view.source("value", .{.x}).slice(.x, .{ .end = 2 }), - view.source("second", .{.x}).slice(.x, .{ .start = 2 }), - }, .x); - const second = view.source("second", .{.x}); - const inputs = [_]Source.Placed{ assembled.withPartitioning(.replicated, .{}), second.withPartitioning(.replicated, .{}) }; - const tensor = Source.bindProgram(&inputs, second.shape(), Sub{}); - const compute = fixture.store.bindingOf(tensor.id).?.compute; - try std.testing.expectEqual(2, compute.inputs.len); - try std.testing.expect(compute.program.as(Sub) != null and compute.program.as(LoaderTestFixture.Identity) == null); - try std.testing.expect(compute.inputs[0].whole() == null); - try std.testing.expect(compute.inputs[1].whole() != null); - var loader = try Loader.init(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - var output: Buffer = undefined; - try loader.start(&tensor, &output, &fixture.store, .none); - try loader.wait(); - defer output.deinit(); - // { 1, 2, 7, 8 } - { 5, 6, 7, 8 }, wrapping. - try testing.expectShards(allocator, io, &output, &.{ 252, 252, 0, 0 }, .default); -} - -test "a padded host-mapped table stays in host memory and serves device gathers in place" { - // The engram arrangement: one source of 264-byte rows, an odd row count - // padded to a multiple of 8, rows sharded over every device, delivered - // as device-kind views of pinned host memory. A program gathers rows - // from each device's local shard. - try expectHostTableGathers(.rows); -} - -test "a replicated host-mapped table is one allocation that every device reads in place" { - // The same table on every device: the direct loader makes it one - // allocation (`direct_loader.DestinationState.initHost`), each device's - // shard a view of it, and every device's kernels gather from it. - try expectHostTableGathers(.replicated); -} - -/// Loads a padded table as `host_mapped`, rows sharded over every device or -/// replicated on each, and gathers rows of each device's shard on that -/// device. The table is freed while the gather is queued behind matrix -/// products: its memory must outlive the read, and go once it is done. -fn expectHostTableGathers(placement: enum { rows, replicated }) !void { - const allocator = std.testing.allocator; - const io = std.testing.io; - const rows: usize = 100_003; - const row_bytes: usize = 264; - const gathered: usize = 256; - - // The accelerator under test, or four CPU devices. - const shared = testing.env(); - const owned: ?*Platform = if (shared.target == .cpu) - try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = 4 } }) - else - null; - defer if (owned) |platform| platform.deinit(allocator, io); - const platform: *const Platform = if (owned) |p| p else shared; - const api = platform.pjrt_api; - const mesh: *const Sharding.Mesh = if (owned) |p| - try p.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) - else - shared.meshes.get("model").?; - if (!backend.supportsMemory(platform, .host_mapped)) return error.SkipZigTest; - const devices = mesh.devicesInCanonicalOrder(); - const shard_count = devices.len; - - const padded_rows = std.mem.alignForward(usize, rows, 8); - const shard_rows = if (placement == .rows) padded_rows / shard_count else padded_rows; - if (placement == .rows) try std.testing.expectEqual(padded_rows, shard_rows * shard_count); - const padded = try allocator.alloc(u8, padded_rows * row_bytes); - defer allocator.free(padded); - @memset(padded, 0); - for (0..rows) |row| { - const bytes = padded[row * row_bytes ..][0..row_bytes]; - // The row's number first, so a misplaced row cannot pass for another. - std.mem.writeInt(u32, bytes[0..4], @intCast(row + 1), .little); - for (bytes[4..], 4..) |*byte, column| byte.* = @as(u8, @truncate(row *% 131 +% column *% 7)) | 1; - } - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try testing.writeSafetensors(allocator, io, tmp.dir, "table.safetensors", ®istry, &.{ - .{ .name = "table", .dtype = "U8", .dims = &.{ @intCast(rows), @intCast(row_bytes) }, .bytes = padded[0 .. rows * row_bytes] }, - }); - var store: TensorStore = .init(allocator, ®istry, .{ .meshes = &.{mesh} }); - defer store.deinit(); - const source = store.view().source("table", .{ .row, .channel }) - .pad(0, .{ .row = TensorStore.Source.Pad{ .high = @intCast(padded_rows - rows) } }); - const table = switch (placement) { - .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), - .replicated => source.withPartitioning(.model, .{}).bind(), - }; - store.view().setMemory(table, .host_mapped); - try std.testing.expectEqual(@as(i64, @intCast(padded_rows)), table.dim(.row)); - - const live_before = direct_loader.liveHostAllocations(); - var loader = try Loader.init(allocator, io, platform, .{ .read_parallelism = 4 }); - defer loader.deinit(); - // The device allocators' counts, once the set-up's calibration has - // allocated on the devices. Bytes in use would not do: CUDA's counts - // what its pool holds, which freed memory leaves room in. CPU's plugin - // counts nothing. - try loader.awaitSetup(); - var allocations: [Platform.MAX_NUM_DEVICES]?u64 = undefined; - for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; - var device_table: Buffer = undefined; - try loader.start(&table, &device_table, &store, .none); - try loader.wait(); - var freed = false; - defer if (!freed) device_table.deinit(); - // The table took no device memory: nothing was allocated on a device. - for (devices) |device| if (allocations[device.id]) |before| { - try std.testing.expectEqual(before, platform.devices[device.id].memoryStats().num_allocs.?); - }; - try std.testing.expectEqual(padded.len, loader.stats.bytes_loaded); - try testing.expectShards(allocator, io, &device_table, padded, .host_mapped); - // A row shard is one allocation per device, a replicated table one. - try std.testing.expectEqual(live_before + if (placement == .rows) shard_count else 1, direct_loader.liveHostAllocations()); - - const first_pointer = try device_table._shards.get(0).opaqueDeviceMemoryDataPointer(api); - for (devices, device_table._shards.constSlice()) |device, shard| { - // Replicas share one allocation; row shards never do. - const pointer = try shard.opaqueDeviceMemoryDataPointer(api); - const shares = placement == .replicated and shard_count > 1; - if (device.id != devices[0].id) try std.testing.expectEqual(shares, pointer == first_pointer); - } - - // Each device gathers `gathered` rows of its own shard, the padding - // rows of the last one included (of every replica, replicated), once - // a chain of matrix products is done: the indices depend on them. - const Gather = struct { - fn forward(table_: Tensor, indices: Tensor, ballast: Tensor) Tensor { - var product = ballast; - for (0..8) |_| product = product.dot(product.withTags(.{ .k, .j }), .k).withTags(.{ .i, .k }); - // Zero, as every product keeps the ballast's positive entries. - const offset: Tensor = .select( - product.sum(.i).sum(.k).reshape(Shape.init(.{}, .f32)).cmp(.LT, .scalar(0, .f32)), - .scalar(1, .u32), - .scalar(0, .u32), - ); - const output: Shape = Shape.init(.{ .shard = indices.dim(.shard), .bytes = indices.dim(.k) * table_.dim(.channel) }, .u8) - .withPartitioning(table_.shape()._sharding.mesh.?, .{ .shard = .model }); - return ops.manualComputation((struct { - fn body(inputs: struct { table: Tensor, indices: Tensor, offset: Tensor }, local: Shape) Tensor { - const shifted = inputs.indices.add(inputs.offset.broad(inputs.indices.shape())); - return inputs.table.gather(.{ .row = shifted }, .{}).reshape(local); - } - }).body, .{ .table = table_, .indices = indices, .offset = offset }, output); - } - }; - const indices_shape = Shape.init(.{ .shard = shard_count, .k = gathered }, .u32).withPartitioning(mesh, .{ .shard = .model }); - const indices = try allocator.alloc(u32, shard_count * gathered); - defer allocator.free(indices); - for (0..shard_count) |shard| { - const local = indices[shard * gathered ..][0..gathered]; - for (local, 0..) |*index, k| index.* = @intCast((k * 7919 + shard * 104_729) % shard_rows); - // The first and last rows of every shard; on the last device these - // are padding. - local[0] = 0; - local[1] = @intCast(shard_rows - 1); - local[2] = @intCast(shard_rows - 2); - } - var indices_buffer: Buffer = try .from(io, platform, indices_shape, std.mem.sliceAsBytes(indices), .{}); - defer indices_buffer.deinit(); - // Milliseconds of products on an accelerator; a token amount on CPU. - const ballast_dim: usize = if (platform.target == .cpu) 32 else 4096; - const ballast_shape = Shape.init(.{ .i = ballast_dim, .k = ballast_dim }, .f32).withPartitioning(mesh, .{}); - const ballast = try allocator.alloc(f32, ballast_dim * ballast_dim); - defer allocator.free(ballast); - @memset(ballast, 1.0 / @as(f32, @floatFromInt(ballast_dim))); - var ballast_buffer: Buffer = try .from(io, platform, ballast_shape, std.mem.sliceAsBytes(ballast), .{}); - defer ballast_buffer.deinit(); - - const exe = try platform.compileFn(allocator, io, Gather.forward, .{ Tensor.fromShape(table.shape()), Tensor.fromShape(indices_shape), Tensor.fromShape(ballast_shape) }, .{ .meshes = &.{mesh} }); - defer exe.deinit(); - var args = try exe.args(allocator); - defer args.deinit(allocator); - var results = try exe.results(allocator); - defer results.deinit(allocator); - args.set(.{ device_table, indices_buffer, ballast_buffer }); - try exe.call(io, args, &results, .{}); - var output = results.get(Buffer); - defer output.deinit(); - - // The table goes while the gather may still have to read it. Pinned - // buffers of its size and another pattern would take its memory had it - // gone early, and the gather would return their bytes. - device_table.deinit(); - freed = true; - const decoy_bytes = try allocator.alloc(u8, padded.len); - defer allocator.free(decoy_bytes); - @memset(decoy_bytes, 0xee); - var decoy: Buffer = try .from(io, platform, table.shape(), decoy_bytes, .{ .memory = .host_pinned }); - defer decoy.deinit(); - - const expected = try allocator.alloc(u8, shard_count * gathered * row_bytes); - defer allocator.free(expected); - for (0..shard_count) |shard| { - for (0..gathered) |k| { - const first_row = if (placement == .rows) shard * shard_rows else 0; - const row = first_row + indices[shard * gathered + k]; - @memcpy(expected[(shard * gathered + k) * row_bytes ..][0..row_bytes], padded[row * row_bytes ..][0..row_bytes]); - } - } - try testing.expectShards(allocator, io, &output, expected, .default); - - // The allocations go with their last views, once nothing reads them; - // PJRT may drop the views on its own threads just after the output is - // ready. - var polls: usize = 0; - while (direct_loader.liveHostAllocations() != live_before) : (polls += 1) { - if (polls == 5000) return error.TestUnexpectedResult; - try io.sleep(.fromMilliseconds(1), .awake); - } - - // The control: the same table in device memory is allocated on every - // device, so the check after the load above can fail. - const device_copy = switch (placement) { - .rows => source.withPartitioning(.model, .{ .row = .model }).bind(), - .replicated => source.withPartitioning(.model, .{}).bind(), - }; - for (devices) |device| allocations[device.id] = platform.devices[device.id].memoryStats().num_allocs; - var device_buffer: Buffer = undefined; - try loader.start(&device_copy, &device_buffer, &store, .none); - try loader.wait(); - defer device_buffer.deinit(); - for (devices) |device| if (allocations[device.id]) |before| { - try std.testing.expect(platform.devices[device.id].memoryStats().num_allocs.? > before); - }; -} - -/// A loader on the buffered backend, whichever backend `platform` loads -/// through (`backend.loadsDirect`), its set-up done in place: CPU loads -/// through the direct backend, and this keeps the front end over the -/// buffered one under test. -fn initBuffered(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Loader.Options) !Loader { - const backend_opts = try backendOptions(io, opts); - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .setup = null, - .backend = .{ .buffered = try buffered_loader.Loader.create(allocator, io, platform, backend_opts) }, - .program_group_bytes = opts.program_group_bytes, - }; -} - -test "the front end loads through the buffered backend and fails with it" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var fixture: LoaderTestFixture = undefined; - try fixture.init(allocator, io); - defer fixture.deinit(); - var loader = try initBuffered(allocator, io, fixture.platform, .{ .read_parallelism = 2 }); - defer loader.deinit(); - - // A tensor, one assembled from value's first half and second's last - // half, one padded with zeros, and a program tensor, in a group of its - // own. - const Source = TensorStore.Source; - const view = fixture.store.view(); - const Model = struct { value: Tensor, assembled: Tensor, padded: Tensor, computed: Tensor }; - const model: Model = .{ - .value = fixture.value, - .assembled = Source.concatenate(&.{ - view.source("value", .{.x}).slice(.x, .{ .end = 2 }), - view.source("second", .{.x}).slice(.x, .{ .start = 2 }), - }, .x).withPartitioning(.replicated, .{}).bind(), - .padded = view.source("second", .{.x}).pad(0, .{ .x = Source.Pad{ .high = 2 } }).withPartitioning(.replicated, .{}).bind(), - .computed = fixture.computed("value", fixture.value, LoaderTestFixture.Add{ .value = 1 }), - }; - var buffers = try mem.bufferize(allocator, &model); - defer mem.deinitBufferized(allocator, &buffers); - try loader.start(&model, &buffers, &fixture.store, .none); - errdefer loader.cancel(); - try loader.wait(); - try testing.expectShards(allocator, io, &buffers.value, &LoaderTestFixture.contents, .default); - try testing.expectShards(allocator, io, &buffers.assembled, &.{ 1, 2, 7, 8 }, .default); - try testing.expectShards(allocator, io, &buffers.padded, &.{ 5, 6, 7, 8, 0, 0 }, .default); - try testing.expectShards(allocator, io, &buffers.computed, &.{ 2, 3, 4, 5 }, .default); - - // A missing file fails the loader, and every later call with it. - var broken: Buffer = undefined; - try loader.start(&fixture.missing, &broken, &fixture.store, .none); - defer broken.deinit(); - try std.testing.expectError(error.FileNotFound, loader.wait()); - try LoaderTestFixture.expectFailed(&loader, error.FileNotFound); - try std.testing.expectEqual(0, broken._shards.len); - try std.testing.expectError(error.FileNotFound, loader.start(&fixture.value, &broken, &fixture.store, .none)); -} diff --git a/zml/io/log.zig b/zml/io/log.zig deleted file mode 100644 index e5bb5820..00000000 --- a/zml/io/log.zig +++ /dev/null @@ -1,38 +0,0 @@ -//! The loader's log scopes. A failure is logged once, where it is -//! detected, at `err` with the values that explain it; cancellation and a -//! plain allocation failure log at `debug`. Frames that only propagate an -//! error do not log it again. Test builds log `err` at `debug`, since the -//! test runner counts logged errors as failures: error-path tests assert -//! the returned errors and the cleanup, not the logs. -const std = @import("std"); -const builtin = @import("builtin"); - -pub const load = Scoped(.@"zml/io/load"); -pub const io = Scoped(.@"zml/io"); -pub const mem = Scoped(.@"zml/mem"); - -fn Scoped(comptime scope: @EnumLiteral()) type { - const base = std.log.scoped(scope); - return struct { - pub const debug = base.debug; - pub const info = base.info; - pub const warn = base.warn; - - /// A terminal failure. The test runner counts logged errors as - /// test failures, so test builds log it at `debug` instead. - pub fn err(comptime fmt: []const u8, args: anytype) void { - if (builtin.is_test) base.debug(fmt, args) else base.err(fmt, args); - } - - /// The first failure of a pipeline, with the error's name appended. - /// Cancellation is expected, and an `OutOfMemory` is a plain - /// allocation failure, returned unlogged, or was logged where it - /// failed (host memory): those log at `debug`. - pub fn failure(e: anyerror, comptime fmt: []const u8, args: anytype) void { - const named_fmt = fmt ++ ": {s}"; - const named_args = args ++ .{@errorName(e)}; - const logged = e == error.Canceled or e == error.Cancelled or e == error.OutOfMemory; - if (logged) debug(named_fmt, named_args) else err(named_fmt, named_args); - } - }; -} diff --git a/zml/kernel.zig b/zml/kernel.zig deleted file mode 100644 index 03a64d70..00000000 --- a/zml/kernel.zig +++ /dev/null @@ -1,1614 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -const cute_builder = @import("dsl/cute/builder"); -const fly_builder = @import("dsl/fly/builder"); -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -const triton_builder = @import("dsl/triton/builder"); - -const Compiler = @import("Compiler.zig"); -const DataType = @import("dtype.zig").DataType; -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const platform_ = @import("platform.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub const triton = struct { - pub const Builder = triton_builder.Builder; - pub const Value = triton_builder.Value; - pub const DType = triton_builder.DType; - pub const FinishError = triton_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; - } -}; - -pub const mosaic_tpu = struct { - const Builder = mosaic_tpu_builder.Builder; - const DType = mosaic_tpu_builder.DType; - const FinishError = mosaic_tpu_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; - } - - const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - }; - - const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, - }; - - const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, - }; - - const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - }; - - fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, - ) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); - } - - const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, - }; - - fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); - } -}; - -pub const cuda_tile = struct { - pub const Builder = cuda_tile_builder.Builder; - pub const Value = cuda_tile_builder.Value; - pub const DType = cuda_tile_builder.DType; - pub const FinishError = cuda_tile_builder.FinishError; - pub const EntryHint = cuda_tile_builder.EntryHint; - pub const Arch = cuda_tile_builder.Arch; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const cute = struct { - pub const Builder = cute_builder.Builder; - pub const ArgSpec = cute_builder.ArgSpec; - pub const Value = cute_builder.Value; - pub const View = cute_builder.View; - pub const Atom = cute_builder.Atom; - pub const DType = cute_builder.DType; - pub const AlgebraToken = cute_builder.AlgebraToken; - pub const LayoutSpec = cute_builder.LayoutSpec; - pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; - pub const FinishError = cute_builder.FinishError; - pub const TmaConfig = cute_builder.TmaConfig; - pub const TmaFormat = cute_builder.TmaFormat; - pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } - - fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A complete CuTe program: one or more `cuda.kernel` operations inside - /// a `gpu.module`, followed by a public `func.func` which constructs the - /// launch-time CuTe objects and calls `cuda.launch_ex`. - /// - /// This is the source-level equivalent of invoking a Python `@cute.jit` - /// object whose `__call__` builds TMA descriptors and launches an - /// `@cute.kernel`. XLA executes the generated host function on its CUDA - /// stream, so the custom call deliberately carries no external grid or - /// block dimensions. - pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const tcl = struct { - pub const Builder = tcl_builder.Builder; - pub const TensorOperation = tcl_builder.TensorOperation; - pub const Value = tcl_builder.Value; - pub const Axis = tcl_builder.Axis; - pub const Expr = tcl_builder.Expr; - pub const Dram = tcl_builder.Dram; - pub const Context = tcl_builder.Context; - pub const DType = tcl_builder.DType; - pub const VeOp = tcl_builder.VeOp; - pub const ReduceMode = tcl_builder.ReduceMode; - pub const Predicate = tcl_builder.Predicate; - pub const Tactic = tcl_builder.Tactic; - pub const FinishError = tcl_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A TCL function compiled by the Furiosa plugin as one native unit: - /// `declareArgs` declares the inputs in order, `ret` the outputs. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; - } -}; - -pub const metal = struct { - pub const Builder = metal_builder.Builder; - pub const Value = metal_builder.Value; - pub const Type = metal_builder.Type; - pub const Struct = metal_builder.Struct; - pub const ArgSpec = metal_builder.ArgSpec; - pub const LoopOpts = metal_builder.LoopOpts; - pub const FinishError = metal_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); - } - - pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; - } - - /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host - /// `extern struct` laid out like a `constant S &` argument. - pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } - }; - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; - } - - /// A Metal kernel: `run` declares its arguments with `declareArgs`, by - /// name, and builds its body. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const fly = struct { - /// Threads per wavefront: 64 on CDNA, 32 on RDNA. - pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; - } - - /// The plugin sizes a block in wavefronts, so a thread count has to divide - /// evenly: a kernel's layouts are built for an exact number of threads and - /// spare ones would partition out of range. - fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); - } - - pub const Builder = fly_builder.Builder; - pub const Value = fly_builder.Value; - pub const DType = fly_builder.DType; - pub const FinishError = fly_builder.FinishError; - pub const TiledCopy = fly_builder.TiledCopy; - pub const TiledMma = fly_builder.TiledMma; - pub const Arch = fly_builder.Arch; - pub const MmaFlavor = fly_builder.MmaFlavor; - pub const fly_rocdl = fly_builder.fly_rocdl; - - /// The matrix atom this device provides, or null when it has none: WMMA - /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. - pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; - } - pub const layout = fly_builder.layout; - pub const Layout = fly_builder.Layout; - pub const Tile = fly_builder.Tile; - pub const IntTuple = fly_builder.IntTuple; - pub const L = fly_builder.L; - pub const rowMajor = fly_builder.rowMajor; - pub const colMajor = fly_builder.colMajor; - pub const ordered = fly_builder.ordered; - pub const tile = fly_builder.tile; - pub const it = fly_builder.it; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernels run on rocm" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: Tensor) Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const model = platform.meshes.get("model").?; - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(model, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: Tensor) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); - return .{ r[0], r[1] }; - } - }; - const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{model} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: Tensor, y: Tensor) Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index b5fe7682..81a35cf4 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [flash_attention.zig](flash_attention.zig) compare results +The tests in [tests.zig](tests.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index 0d01c357..ec0fce16 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -1,4 +1,7 @@ const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; @@ -14,6 +17,8 @@ pub const Config = struct { head_dim: i64, dtype: cute.DType, splits: i64 = 1, + /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. + causal: bool = true, }; /// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA @@ -144,13 +149,16 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void const query = row.div(group).minimum(cfg.query_length - 1); const head = kv_head.mul(group).add(row.rem(group)); const start = a.positions.get(.{batch}); - const position = start.add(query); - const causal_end = start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1); + const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); + const live_end = if (cfg.causal) + start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) + else + b.constant(.i32, cfg.key_length); // Partition the live cache, rather than its capacity, so decode uses all // blocks even when only a prefix of the cache is populated. - const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); + const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); - const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; + const end = if (split) begin.add(chunk).minimum(live_end) else live_end; const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); @@ -305,6 +313,494 @@ fn merge(b: *B, cfg: Config) cute.FinishError!void { } } +// ==================== Prefill ==================== +// +// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 +// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one +// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and +// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, +// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied +// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. +// +// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared +// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas +// caps at 48 KiB before sm_120. + +pub const Prefill = cute.Program(Config, .{ + .name = "flash_attention_cute_prefill", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = prefillProgram, +}); + +const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; + +/// Keys per tile (FA2's kBlockN). +const prefill_keys = 64; + +/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. +/// Same as FA2 we only support 16-bit inputs +pub fn prefillSupported(cfg: Config) bool { + const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; + const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; + return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; +} + +/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. +fn prefillRowTiles(head_dim: i64) i64 { + return if (head_dim <= 128) 2 else 1; +} + +/// Queries per block (FA2's kBlockM). +fn prefillRows(cfg: Config) i64 { + return 4 * 16 * prefillRowTiles(cfg.head_dim); +} + +/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. +fn prefillGrid(cfg: Config) [3]i32 { + return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; +} + +fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(prefillSupported(cfg)); + switch (cfg.head_dim) { + inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { + inline .bf16, .f16 => |dt| { + b.beginFunction(prefill_kernel_name, .cuda_kernel); + try prefillSized(b, cfg, d, dt); + b.endFunction(block); + + b.beginFunction(Prefill.name, .host); + const host = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const grid = prefillGrid(cfg); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, + .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), + .stream = b.cudaStream(), + }); + const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); + }, + else => unreachable, + }, + else => unreachable, + } +} + +/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and +/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. +fn fmax(b: *B, x: Value, y: Value) Value { + return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; +} + +fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { + var result: [n]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + } + return result; +} + +/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. +fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { + const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), + .constraints = "=r,=r,=r,=r,r", + .has_side_effects = true, + }, b.loc())); + return unpackStruct(b, 4, result, .i32); +} + +/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). +fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), + .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop +/// carries at most 64 values). +fn packAcc(b: *B, values: [4]Value) Value { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn unpackAcc(b: *B, value: Value) [4]Value { + var result: [4]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.vectorExtract(value, @intCast(i), .f32); + } + return result; +} + +/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. +/// +/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so +/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses +/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a +/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from +/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant +/// (XORed into the 16-byte vector index). +fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { + return struct { + const row_tiles = prefillRowTiles(d); + const rows = 4 * 16 * row_tiles; + const n = prefill_keys; + const score_tiles = n / 8; + const out_tiles = d / 8; + const row_bytes = d * 2; + /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. + const swizzle_rows = @min(8, d / 8); + const q_bytes = rows * row_bytes; + const kv_bytes = n * row_bytes; + /// Rows covered by one 16-byte cp.async per thread of the block. + const copy_rows = @divExact(128, d / 8); + /// Running max and per-thread partial sum of the two rows of every row tile. + const stats = 2 * row_tiles; + /// Carried by the key loop: maxima, sums, then the packed output accumulators of every + /// (row tile, 8 output features). + const carried = 2 * stats + row_tiles * out_tiles; + const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); + + comptime { + // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. + std.debug.assert(std.math.isPowerOfTwo(copy_rows)); + std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); + } + + b: *B, + cfg: Config, + lane: Value, + warp: Value, + first_query: Value, + start: Value, + q_smem: Value, + k_smem: Value, + v_smem: Value, + k_ptr: Value, + v_ptr: Value, + kv_base: Value, + tiles: Value, + /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. + copy_row: Value, + copy_col: Value, + /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. + copy_row_offset: Value, + copy_vector_offset: Value, + /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. + q_frag_row: Value, + k_frag_row: Value, + v_frag_row: Value, + qv_frag_vector: Value, + k_frag_vector: Value, + + const Self = @This(); + + fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { + const lane = tid.rem(32); + const warp = tid.div(32); + const copy_row = tid.div(d / 8); + const copy_col = tid.rem(d / 8); + const lane8 = lane.rem(8); + // The fragment rows a thread reads are lane % 8 plus multiples of 8. + const lane_xor = lane.rem(swizzle_rows); + return .{ + .b = b, + .cfg = cfg, + .lane = lane, + .warp = warp, + .first_query = first_query, + .start = start, + .q_smem = shared, + .k_smem = shared.add(q_bytes), + .v_smem = shared.add(q_bytes + kv_bytes), + .k_ptr = k_ptr, + .v_ptr = v_ptr, + .kv_base = kv_base, + .tiles = tiles, + .copy_row = copy_row, + .copy_col = copy_col, + .copy_row_offset = copy_row.mul(row_bytes), + .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), + // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). + .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), + // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). + .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), + // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). + .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), + .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), + .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), + }; + } + + /// Destination of the i-th cp.async of this thread in a tile starting at `base`. + fn copyDst(t: Self, base: Value, comptime i: usize) Value { + const xor = ((copy_rows * i) % swizzle_rows) * 16; + const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); + return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); + } + + /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. + fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { + const b = t.b; + const stride = t.cfg.kv_heads * d; + const first_key = key_base.add(t.copy_row); + const safe = src.add(t.kv_base.mul(2)); + const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(n, copy_rows)) |i| { + const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); + const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); + b.cpAsync16(t.copyDst(dst, i), source, valid); + } + } + + fn queryOf(t: Self, mi: usize, h: usize) Value { + return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); + } + + fn accIndex(comptime mi: usize, comptime di: usize) usize { + return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); + } + + /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are + /// fully visible to every query of the block. + fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { + const b = t.b; + const lane = t.lane; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const key_base = tile.mul(n); + b.cpAsyncWaitAll(); + b.syncThreads(); + t.loadTile(t.v_ptr, t.v_smem, key_base); + + // S = Q K^T, k-steps of 16 features. + var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); + inline for (0..d / 16) |ks| { + var qa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); + } + const k_vector = t.k_frag_vector.bitXor(ks * 32); + inline for (0..score_tiles / 2) |np| { + const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); + inline for (0..row_tiles) |mi| { + s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); + s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); + } + } + } + + // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: + // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). + var next: [carried]Value = undefined; + var alpha: [row_tiles][2]Value = undefined; + inline for (0..row_tiles) |mi| { + if (masked) { + inline for (0..score_tiles) |ni| { + inline for (0..4) |j| { + const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); + const in_range = key.lt(t.cfg.key_length); + const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; + s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); + } + } + } + inline for (0..2) |h| { + var row_max = s[mi][0][h * 2]; + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); + } + } + inline for (.{ 2, 1 }) |offset| { + row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); + } + const old_max = carried_in[mi * 2 + h]; + const new_max = fmax(b, old_max, row_max); + // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. + const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); + alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); + next[mi * 2 + h] = new_max; + var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); + sum = sum.add(s[mi][ni][h * 2 + j]); + } + } + next[stats + mi * 2 + h] = sum; + } + } + + // All warps are done with K_j once V_j has landed: start copying K_{j+1}. + b.cpAsyncWaitAll(); + b.syncThreads(); + var prefetch = b.openIf(tile.add(1).lt(t.tiles)); + { + t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); + prefetch.close(); + } + + // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. + var o: [row_tiles][out_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); + inline for (0..4) |j| { + o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); + } + } + } + inline for (0..n / 16) |ks| { + var pa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + pa[mi] = .{ + b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), + b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), + b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), + b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), + }; + } + const v_row = t.v_frag_row.add(ks * 16 * row_bytes); + inline for (0..out_tiles / 2) |dp| { + const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); + inline for (0..row_tiles) |mi| { + o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); + o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); + } + } + } + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + next[accIndex(mi, di)] = packAcc(b, o[mi][di]); + } + } + return next; + } + }; +} + +fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { + const T = PrefillTile(d, dt); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + + const tid = b.threadIdx().x; + const first_query = b.blockIdx().x.mul(T.rows); + const head = b.blockIdx().y; + const batch = b.blockIdx().z; + const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); + // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the + // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and + // `allocSmemView` both become static shared memory instead. + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); + const fields = [_][]const u8{ + std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), + std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), + std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), + }; + const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); + const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); + const q_ptr = b.ptrToInt(a.q, .i64); + const out_ptr = b.ptrToInt(a.out, .i64); + const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; + + // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. + const q_stride = cfg.query_heads * d; + const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); + const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); + + // Causal: the last query of the block sees keys up to start + last_query, and every query of the + // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the + // keys, is masked. + const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); + const tiles = if (cfg.causal) + start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) + else + b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); + const full_tiles = if (cfg.causal) + start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) + else + b.constant(.i32, @divTrunc(cfg.key_length, T.n)); + + const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); + + // Out of range rows are zero-filled and never read from global memory. + { + const first_row = first_query.add(t.copy_row); + const safe = q_ptr.add(q_base.mul(2)); + const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(T.rows, T.copy_rows)) |i| { + const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); + b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); + } + } + t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); + + var initial: [T.carried]Value = undefined; + const zero = b.constant(.f32, 0); + for (initial[0..T.stats]) |*v| { + v.* = b.constant(.f32, -std.math.inf(f32)); + } + for (initial[T.stats .. 2 * T.stats]) |*v| { + v.* = zero; + } + for (initial[2 * T.stats ..]) |*v| { + v.* = packAcc(b, .{ zero, zero, zero, zero }); + } + + var full = b.openFor(0, full_tiles, 1, initial); + { + full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); + } + var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); + { + diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); + } + const results = diagonal.results[0..T.carried].*; + + // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. + inline for (0..T.row_tiles) |mi| { + inline for (0..2) |h| { + var sum = results[T.stats + mi * 2 + h]; + inline for (.{ 2, 1 }) |offset| { + sum = sum.add(b.shuffleXor(sum, offset)); + } + const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); + const query = t.queryOf(mi, h); + var valid = b.openIf(query.lt(cfg.query_length)); + { + const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); + inline for (0..T.out_tiles) |di| { + const acc = unpackAcc(b, results[T.accIndex(mi, di)]); + const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); + b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); + } + valid.close(); + } + } + } +} + test "CuTe attention emits tiled online softmax through head dimension 512" { for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { const ir = try Kernel.emit(std.testing.allocator, .{ diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 2958888d..36e0953b 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -15,7 +15,8 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major >= 8; } -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor) zml.Tensor { +/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); const batched = q.shape().hasTag(.b) != null; const rank: usize = if (batched) 4 else 3; @@ -24,7 +25,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); - stdx.debug.assert(q.dim(.q) > 0 and q.dim(.q) <= k.dim(.k) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq <= Lk and matching K/V lengths", .{}); + stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); if (batched) { stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); @@ -33,64 +34,51 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); } - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(mesh, .{ .h = .model }); - const k_sharded = k.withPartitioning(mesh, .{ .h = .model }); - const v_sharded = v.withPartitioning(mesh, .{ .h = .model }); const positions = if (batched) token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q_sharded.shape()); -} + const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); + const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); + const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - positions: zml.Tensor, + const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const batched = self.q.shape().hasTag(.b) != null; - const q = if (batched) self.q.transpose(.{ .b, .q, .h, .hd }) else self.q.transpose(.{ .q, .h, .hd }); - const k = if (batched) self.k.transpose(.{ .b, .k, .h, .hd }) else self.k.transpose(.{ .k, .h, .hd }); - const v = if (batched) self.v.transpose(.{ .b, .k, .h, .hd }) else self.v.transpose(.{ .k, .h, .hd }); + const cfg: kernel.Config = .{ + .batch = if (batched) q_.dim(.b) else 1, + .query_length = q_.dim(.q), + .key_length = k_.dim(.k), + .query_heads = q_.dim(.h), + .kv_heads = k_.dim(.h), + .head_dim = q_.dim(.hd), + .dtype = zml.kernel.cute.from(q_.dtype()), + .splits = splits, + .causal = causal, + }; - const splits = kernel.decodeSplits(q.shape(), k.shape(), decodeResources(zml.Compiler.current().platform)); - - const cfg: kernel.Config = .{ - .batch = if (batched) q.dim(.b) else 1, - .query_length = q.dim(.q), - .key_length = k.dim(.k), - .query_heads = q.dim(.h), - .kv_heads = k.dim(.h), - .head_dim = q.dim(.hd), - .dtype = zml.kernel.cute.from(q.dtype()), - .splits = splits, - }; - - const out = if (cfg.splits > 1) b: { - const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ - .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), - }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, - .block = kernel.block, - }).out; - break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, - .block = kernel.block, - }).out; - } else kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ .cfg = cfg, - .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, .block = kernel.block, }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; - return out.transpose(output_shape); - } -}; + return out.transpose(q.shape()); +} /// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. /// If there are no devices, returns 0 for both values. diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index c1a27a36..3fb898db 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,9 +6,11 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -126,239 +128,12 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { + zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { - .cute => cute.attention(q, k, v, token_index), - .nki => nki.attention(q, k, v, token_index, parameters.nki), - .metal => metal.attention(q, k, v, token_index, metadata.metal), + .cute => cute.attention(q, k, v, token_index, opts.causal), + // The NKI and Metal kernels are causal only. + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, std.enums.values(Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, backends: []const Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - for (token_index_h) |index| { - // Check for out of bound reads - std.debug.assert(index + max_q_idx <= max_k_idx); - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, zml.attention.attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: Metadata = .init(.fromBackend(backend)); - const parameters: Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index 855bdcca..f82ade45 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = ["metal.zig"], + srcs = [ + "flash_attn.zig", + "metal.zig", + ], ) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig new file mode 100644 index 00000000..de4b68da --- /dev/null +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -0,0 +1,732 @@ +//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` +//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, +//! fa_tiled otherwise. + +const std = @import("std"); + +const metal = @import("../../../kernel/kernel.zig").metal; + +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVec = struct { + pub const Cfg = struct { + head_dim: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "k_len", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + k_len: i32, + scale: f32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec", + .inputs = &.{ "q", "k", "v", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k = char_in, + .v = char_in, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const n_kv = a.args.field("n_kv"); + const group = num_heads.div(n_kv); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + const kv_len = a.token_index.load(r).add(1); + const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const position_bytes = n_kv.to(.ulong).mul(D * 2); + const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); + const k0 = a.k.add(first); + const v0 = a.v.add(first); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, iwg); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first_row) |*scope| scope.close(); + } +}; + +fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { + return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); +} + +/// fa_vec_reduce: merges the fa_vec split partials. +pub const FaVecReduce = struct { + pub const Cfg = struct { + splits: i32, + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_reduce", + .inputs = &.{ "htmp", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVec.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + const live = liveSplits(b, a.token_index.load(r).add(1), NWG); + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past +/// `live` output zeros. +pub const FaTiled = struct { + pub const Cfg = struct { + dtype: Type, + head_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_tiled", + .inputs = &.{ "q", "k", "v", "token_index", "live" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, + .run = run, + }); + + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k = .{ .buffer = .{ .type = T, .const_ = true } }, + .v = .{ .buffer = .{ .type = T, .const_ = true } }, + .token_index = int_in, + .live = int_in, + .num_kv_heads = int_arg, + .scale = .{ .constant = .float }, + .q_len = int_arg, + .k_len = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_pos_start = a.tgp.y().to(.int).mul(BQ); + const batch = a.tgp.z().to(.int); + const num_heads = a.tgpg.x().to(.int); + const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); + const q_stride = num_heads.mul(HEAD_SIZE); + const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); + + const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); + const live = a.live.load(0); + const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); + const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + + var dead = b.openIf(valid_q.le(0)); + { + var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = zero.iv; + out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); + zero.close(); + } + b.return_(); + dead.close(); + } + + // Query i of the batch row is at position context_len + i. + const context_len = a.token_index.load(batch); + const seq_len = context_len.add(live); + const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + var past = b.openIf(tile_start.gt(last_q_pos)); + { + tiles.breakLoop(); + past.close(); + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); + const k_ptr = a.k.add(off); + const v_ptr = a.v.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj).mul(scale_log2); + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); + out_base.store(r.mul(q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the flash attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = FaTiled.tileConfig(head_size).?; + allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); + } + } +} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index 9327c2a1..bdfb00c0 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,12 +1,10 @@ const std = @import("std"); -const stdx = @import("stdx"); - const zml = @import("../../../zml.zig"); - -const log = std.log.scoped(.@"zml/attention/metal"); +const flash_attn = @import("flash_attn.zig"); pub const Metadata = struct { + /// Live queries of a single-sequence prefill; 0 means all. num_tokens: zml.Tensor, pub fn init() Metadata { @@ -23,16 +21,123 @@ pub const Metadata = struct { } }; +/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for +/// unsupported dtypes and head sizes. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const qc = q.transpose(.{ .h, .q, .hd }); - const kc = k.transpose(.{ .h, .k, .hd }); - const vc = v.transpose(.{ .h, .k, .hd }); - const tok_i32 = token_index.convert(.i32); + const dtype = q.dtype(); + const head_dim = q.dim(.hd); + if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { + const seq_len = k.dim(.k); + const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); + return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + } - const attn = if (q.dim(.q) > 1) - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + const batched = q.shape().hasTag(.b) != null; + const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); + const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); + const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); + + const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; + const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) + decode(qb, kb, vb, positions, scale) else - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); + return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); +} - return attn.transpose(q.shape()); +fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { + const all: zml.Tensor = .scalar(q.dim(.q), .i32); + if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; + const num_tokens = metadata.num_tokens.convert(.i32); + return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); +} + +/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { + const batch: i32 = @intCast(q.dim(.b)); + const num_heads: i32 = @intCast(q.dim(.h)); + const head_dim: i32 = @intCast(q.dim(.hd)); + const num_kv_heads: i32 = @intCast(k.dim(.h)); + const keys: i32 = @intCast(k.dim(.k)); + const nrows = batch * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); + const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; + + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ + .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; +} + +/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { + const queries: i32 = @intCast(q.dim(.q)); + const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; + const cfg: flash_attn.FaTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + }; + return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), + .scale = .of(scale), + .q_len = .of(queries), + .k_len = .of(@as(i32, @intCast(k.dim(.k)))), + }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} + +test { + std.testing.refAllDecls(flash_attn); } diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 282da132..f98a545d 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,71 +43,17 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - // TODO: https://github.com/zml/zml/issues/914 - const tp = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(tp, .{ .h = .model }); - const k_sharded = k.withPartitioning(tp, .{ .h = .model }); - const v_sharded = v.withPartitioning(tp, .{ .h = .model }); - - if (q.dim(.q) == 1) { - const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(tp, .{ .row = .replicated, .col = .replicated }); - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - token_index: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.decode_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v, self.token_index }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ - .q = q_sharded, - .k = k_sharded, - .v = v_sharded, - .token_index = token_index_2d, - .parameters = parameters, - }, - q_sharded.shape(), - ).convert(q.dtype()); - } - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.prefill_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, - q_sharded.shape(), - ).convert(q.dtype()); + const decode = q.dim(.q) == 1; + const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; + const options: zml.ops.NeuronNkiOps = .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = parameters.compiler_target, + }; + const output = if (decode) + zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] + else + zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; + return output.convert(q.dtype()); } diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..c3c3dca2 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,346 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { + try testAttention( + .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), + .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), + &.{3000}, + ); +} + +test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), + .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), + &.{900}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig new file mode 100644 index 00000000..b66593c1 --- /dev/null +++ b/zml/kernels/moe/api.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_8 => blk: { + const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); + break :blk .{ + grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + }; + }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { + concatenated, // G0 G1 … Gn U0 U1 … Un + row_interleaved_1, // G0 U0 G1 U1 … + row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … +}; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Gate/up ordering before applying the backend-specific packing. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte + /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, + /// represented as `[expert, k/128, rows]` i32 words of four scales. + mxfp4_interleaved8, + }; +}; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index c0156720..8bc03c9c 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,6 +3,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -24,7 +27,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -35,14 +38,14 @@ pub fn validateOptions(opts: zml.moe.Options) void { /// is the per-expert multiplier dequantizing each GEMM output (it folds in /// the inverse of the input's global scale), and their input scale is the /// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. +/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows +/// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); @@ -83,24 +86,43 @@ pub fn fusedExperts( else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), }; - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .topk = ids.dim(.topk), - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - .format = format, - .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), - }; - return if (context.expert_parallel) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + return switch (format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .ids = ids, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + }), + }.reshape(input.shape().dims()).withTags(input.shape()); } /// The format-specific tensors: NVFP4's per-expert global scales. They stay @@ -115,69 +137,3 @@ const Format = union(enum) { act2: zml.Tensor, }, }; - -const Context = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, - topk: i64, - swiglu_limit: ?f32, - routing_weight_placement: zml.moe.RoutingWeightPlacement, - format: Format, - expert_parallel: bool, - - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; - const tokens = self.input.dim(.t); - var ids = self.ids.convert(.i32); - const routing_weights = self.weights.convert(.f32); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(experts); - const expert_end = expert_start.addConstant(experts); - // Routes of other ranks get expert -1 and are not computed here. - const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); - ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); - } - const result = switch (self.format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ - .x = self.input, - .routing_weights = routing_weights, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ - .x = self.input, - .routing_weights = routing_weights, - .ids = ids, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = self.swiglu_limit, - .routing_weight_placement = self.routing_weight_placement, - }), - }.withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; - } -}; diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig index 47e05d37..da639c75 100644 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -138,7 +138,7 @@ pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) Pipeli pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { var peek = b.openIfElse(count.lt(limit), .{.i1}); { - peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); } { peek.yieldElse(.{b.constant(.i1, true)}); diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index 479e6516..ff6edec2 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,26 +4,31 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); -const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. x: zml.Tensor, /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. w1: zml.Tensor, - /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, + /// in MN-major 128x4 atoms, with rows interleaved like `w1`. s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. w2: zml.Tensor, - /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, @@ -207,14 +212,14 @@ pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, t .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, - .weight_scale = a.s1.bitCast(.u8), + .weight_scale = a.s1, .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, - .weight_scale = a.s2.bitCast(.u8), + .weight_scale = a.s2, .input_scale = q2.s, }, routing.route_inverse); return reduce.topk(down, routing.route_map orelse routing.schedule, .{ @@ -252,7 +257,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index e8901fb1..30062a3a 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -550,7 +550,7 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .weight = .{ .ptr = cute.DType.i8 }, .schedule = .{ .ptr = cute.DType.i32 }, .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i32 }, .input_scale = .{ .ptr = cute.DType.i8 }, }; const shape = .{ @@ -762,10 +762,10 @@ fn issueTmaStage( b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); } - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -826,10 +826,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, k_tiles, @@ -845,7 +842,7 @@ fn runTmaProducer( const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); @@ -893,11 +890,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -922,8 +915,8 @@ const Mxfp4Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1164,17 +1157,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1192,7 +1182,7 @@ fn runMmaConsumer( const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); @@ -1209,7 +1199,7 @@ fn runMmaConsumer( ); var commit_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); } const next_count = stages.carried[2].add(1); @@ -1220,7 +1210,7 @@ fn runMmaConsumer( var full_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); } const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); @@ -1271,11 +1261,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1349,7 +1335,7 @@ fn runEpilogue( .phase = tiles.after_carried[5], }; const stage = accumulator_state.index; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1375,7 +1361,7 @@ fn runEpilogue( if (chunk + 1 == tile_chunks) { // Registers now own the accumulator values, so all 128 epilogue // threads release this TMEM stage before the stores. - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); @@ -1405,7 +1391,7 @@ fn runEpilogue( b.relinquishTmemAllocPermit(.{}); release_warp.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); { b.deallocTmem(tmem, tile.tmem_columns, .{}); @@ -1453,11 +1439,10 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, return .{ .count = count, .route = route, .weight = weight }; } -/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` -/// of the interleaved projection, so even threads hold a gate and their odd -/// neighbour the matching up value. Each even thread computes one SwiGLU -/// activation per routed row; 32 consecutive activations span two warps, -/// whose partial maxima meet in shared memory. +/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 +/// projection. Gate and up lanes are paired by XOR 8. Each gate thread +/// computes one SwiGLU activation per routed row; 32 consecutive activations +/// span two warps, whose partial maxima meet in shared memory. /// /// At decode sizes most rows of a group are padding, whose outputs are never /// read. `count` is uniform across the CTA, so whole 8-row chunks of padding @@ -1484,7 +1469,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const tid = b.threadIdx().x; const lane = tid.rem(32); const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(2).eq(0); + const is_gate = tid.rem(16).lt(8); const first_row = group.mul(operands.group_rows); const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. @@ -1511,7 +1496,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo if (operands.swiglu_limit) |limit| { gate = gate.minimum(limit); } - var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { up_value = up_value.maximum(-limit).minimum(limit); } @@ -1520,7 +1505,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Odd threads hold no activation and must not raise the maxima. + // Up lanes hold no activation and must not raise the maxima. activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); @@ -1545,16 +1530,17 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo scratch.set(.{ warp, lane }, lane_maximum); owner.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); // Lane `c` turns the two half-block maxima of column `c` into its scale. const pair = warp.bitAnd(~@as(i32, 1)); const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); // The next tile reuses the scratch maxima. - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); - const activation_index = tile_m.mul(64).add(tid.div(2)); + const activation_row = tid.div(16).mul(8).add(tid.rem(8)); + const activation_index = tile_m.mul(64).add(activation_row); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1767,7 +1753,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var mma_warp = b.openIf(warp.eq(4)); { // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (k_tiles) { inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), else => unreachable, @@ -1784,7 +1770,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); } - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (epilogue) { .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ .group_sizes = view(b, d.group_sizes), @@ -1844,7 +1830,13 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + // Weight scales step M atoms first; activation scales retain K-major order. + const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); + const sfa_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, + ); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig index ac59735c..0de45007 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -699,10 +699,10 @@ fn issueTmaStage( var elected = b.openIf(b.electSync()); b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -756,10 +756,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, @as(i32, @intCast(k_tiles)), @@ -773,7 +770,7 @@ fn runTmaProducer( }; const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); const next_count = stages.carried[1].add(1); @@ -817,11 +814,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -843,8 +836,8 @@ const Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1020,17 +1013,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1043,7 +1033,7 @@ fn runMmaConsumer( const k = stages.carried[2]; const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); @@ -1058,7 +1048,7 @@ fn runMmaConsumer( mainloop.accumulator, ); var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); const next_count = stages.carried[2].add(1); const next_state = advancePipeline(b, state.index, state.phase, tile.stages); @@ -1066,7 +1056,7 @@ fn runMmaConsumer( stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); const next_acc_count = tiles.after_carried[8].add(1); @@ -1113,11 +1103,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1193,7 +1179,7 @@ fn runEpilogue( const expert = operands.expert_ids.get(.{group}, .i32); const alpha = operands.alpha.get(.{expert}, .f32); const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1213,7 +1199,7 @@ fn runEpilogue( const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); b.fenceTmemLoad(); if (chunk + 1 == tile_chunks) { - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); @@ -1239,7 +1225,7 @@ fn runEpilogue( var release_warp = b.openIf(warp.eq(0)); b.relinquishTmemAllocPermit(.{}); release_warp.close(); - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); b.deallocTmem(tmem, tile.tmem_columns, .{}); dealloc_warp.close(); @@ -1543,7 +1529,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E tma_warp.close(); var mma_warp = b.openIf(warp.eq(4)); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); mma_warp.close(); @@ -1551,7 +1537,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var alloc_warp = b.openIf(warp.eq(0)); b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); const shared_operands: EpilogueOperands = .{ .expert_ids = expert_ids, .alpha = view(b, d.alpha), diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel deleted file mode 100644 index 54b8604c..00000000 --- a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cutlass_flashinfer", - srcs = [ - "cutlass_flashinfer.zig", - ], -) diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig deleted file mode 100644 index 2c7ad342..00000000 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ /dev/null @@ -1,633 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const platforms = @import("platforms"); -const zml = @import("../../../zml.zig"); - -const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); - -pub const auto_tactic: i32 = -1; - -pub const Options = struct { - /// Device used at graph-construction time to select the architecture - /// library and size the XLA-owned scratch buffer. - workspace_query_device: i32, - activation: zml.moe.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to - /// obtain the first valid GEMM2 index. - gemm2_tactic: i32, - - pub fn fromGlobalOptions(opts: zml.moe.Options) Options { - return .{ - .workspace_query_device = 0, - .activation = opts.activation, - .enable_pdl = false, - .gemm1_tactic = auto_tactic, - .gemm2_tactic = auto_tactic, - }; - } -}; - -pub fn validateOptions(opts: zml.moe.Options) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); -} - -const Input = struct { - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, -}; - -const Output = struct { - output: zml.Shape, - workspace: zml.Shape, -}; - -const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; - -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -fn runnersFromAttributes(attributes: Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -fn currentRunners() !*Runners { - const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.fi_cutlass_moe_runners; - return runners orelse error.RunnersNotLoaded; -} - -fn cutlassActivation(activation: zml.moe.Activation) i32 { - return switch (activation) { - .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, - .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, - .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, - .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, - .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, - .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, - .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, - }; -} - -fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} - -fn ffiCallNvfp4( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, -) !?*zml.pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = input.hidden_states.ptr; - io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); - io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); - io.fc1_expert_weights = input.fc1_weights.ptr; - io.fc2_expert_weights = input.fc2_weights.ptr; - io.quant_scales[0] = input.fc1_act_global.ptr; - io.quant_scales[1] = input.fc1_weight_block.ptr; - io.quant_scales[2] = input.fc1_global.ptr; - io.quant_scales[3] = input.fc2_act_global.ptr; - io.quant_scales[4] = input.fc2_weight_block.ptr; - io.quant_scales[5] = input.fc2_global.ptr; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = output.output.ptr; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = output.workspace.ptr; - workspace.data_bytes = output.workspace.shape.byteSize(); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), - ); - return null; -} - -const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ - .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert mesh is owned by forwardMoe outer manual computation. - .sharding_aware = false, - .has_side_effect = false, -}); - -fn computeCapability(platform: *const zml.Platform) !u16 { - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; - return switch (cc.sm()) { - 100, 103, 120 => |sm| sm, - else => error.UnsupportedArchitecture, - }; -} - -pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, -) !void { - if (comptime platforms.isEnabled(.cuda)) { - var cuda_state = &platform.state.cuda; - if (cuda_state.fi_cutlass_moe_runners != null) return; - try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); - const runners = try allocator.create(Runners); - runners.* = .{}; - cuda_state.fi_cutlass_moe_runners = runners; - return; - } - return error.UnsupportedPlatform; -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try routedNvfp4Call.register(platform); - return; - } - return error.UnsupportedPlatform; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; - _ = computeCapability(platform) catch return false; - return true; -} - -pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; - const compute_capability = computeCapability(platform) catch return false; - return switch (compute_capability) { - 100, 103, 120 => true, - else => false, - }; -} - -pub fn tacticCounts( - platform: *const zml.Platform, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - -fn roundUp(value: i64, alignment: i64) i64 { - return @divTrunc(value + alignment - 1, alignment) * alignment; -} - -pub fn fc1BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, - activation: zml.moe.Activation, -) zml.Shape { - const rows = switch (activation) { - .silu, .gelu, .relu => intermediate_size, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, - }; - return .init( - .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, - .f8e4m3fn, - ); -} - -pub fn fc2BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, -) zml.Shape { - return .init( - .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, - .f8e4m3fn, - ); -} - -fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { - return tensor.dtype() == .f32 and - (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); -} - -fn validateInputs( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) !Attributes { - if (hidden_states.dtype() != .bf16 or - fc1_weights.dtype() != .f4e2m1 or - fc2_weights.dtype() != .f4e2m1 or - topk_weights.dtype() != .f32 or - topk_ids.dtype() != .i32 or - fc1_weight_block.dtype() != .f8e4m3fn or - fc2_weight_block.dtype() != .f8e4m3fn or - fc1_global.dtype() != .f32 or - fc2_global.dtype() != .f32) - { - return error.UnsupportedType; - } - if (hidden_states.rank() != 2 or - fc1_weights.rank() != 3 or - fc2_weights.rank() != 3 or - topk_weights.rank() != 2 or - topk_ids.rank() != 2) - { - return error.InvalidShape; - } - - const tokens = hidden_states.dim(.t); - const hidden_size = hidden_states.dim(.d); - const num_experts = fc1_weights.dim(0); - const fc1_rows = fc1_weights.dim(1); - const intermediate_size = switch (options.activation) { - .silu, .gelu, .relu => fc1_rows, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), - }; - const top_k = topk_ids.dim(.topk); - - if (tokens <= 0 or hidden_size <= 0 or - num_experts <= 0 or intermediate_size <= 0 or - @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or - top_k <= 0 or top_k > num_experts) - { - return error.InvalidShape; - } - if (fc1_weights.dim(2) != hidden_size or - fc2_weights.dim(0) != num_experts or - fc2_weights.dim(1) != hidden_size or - fc2_weights.dim(2) != intermediate_size or - topk_weights.dim(.t) != tokens or - topk_weights.dim(.topk) != top_k or - topk_ids.dim(.t) != tokens or - !fc1_weight_block.shape().eql(fc1BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - options.activation, - )) or - !fc2_weight_block.shape().eql(fc2BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - )) or - !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or - !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or - !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or - !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) - { - return error.InvalidShape; - } - - return .{ - .num_tokens = tokens, - .hidden_size = hidden_size, - .intermediate_size = intermediate_size, - .num_experts = @intCast(num_experts), - .top_k = @intCast(top_k), - .activation = cutlassActivation(options.activation), - .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = fc1_act_global.rank() == 1, - .fc2_act_per_expert = fc2_act_global.rank() == 1, - .gemm1_tactic = options.gemm1_tactic, - .gemm2_tactic = options.gemm2_tactic, - }; -} - -/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. -/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout -/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. -pub fn fusedExpertsNvfp4( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) zml.Tensor { - const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); - var attributes = validateInputs( - hidden_states, - fc1_weights, - fc2_weights, - topk_weights, - topk_ids, - fc1_act_global, - fc1_weight_block, - fc1_global, - fc2_act_global, - fc2_weight_block, - fc2_global, - options, - ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); - - attributes.runners = @intFromPtr(runners); - const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = makeContext(attributes); - var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); - requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - checkStatus( - deviceRunner.api, - deviceRunner.api.getWorkspaceRequirements( - deviceRunner.runner, - &context, - &requirements, - ), - ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - - const result = routedNvfp4Call.call( - .{ - .hidden_states = hidden_states, - .fc1_weights = fc1_weights, - .fc2_weights = fc2_weights, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .fc1_act_global = fc1_act_global, - .fc1_weight_block = fc1_weight_block, - .fc1_global = fc1_global, - .fc2_act_global = fc2_act_global, - .fc2_weight_block = fc2_weight_block, - .fc2_global = fc2_global, - }, - .{ - .output = hidden_states.shape(), - .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), - }, - attributes, - ); - return result.output; -} - -pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - if (comptime !platforms.isEnabled(.cuda)) { - @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); - } - - if (gate_up.bias != null or down.bias != null) { - @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); - } - - const runner_options: Options = .fromGlobalOptions(opts); - const expert_partition = gate_up.weight.shape().partition(.expert); - - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.isSharded()) { - return zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up_weight_unpacked: zml.Tensor, - down_weight_unpacked: zml.Tensor, - gate_up_input_scale: zml.Tensor, - gate_up_scales: zml.Tensor, - gate_up_global_scale: zml.Tensor, - down_input_scale: zml.Tensor, - down_scales: zml.Tensor, - down_global_scale: zml.Tensor, - activation: zml.moe.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = fusedExpertsNvfp4( - self.input, - self.gate_up_weight_unpacked, - self.down_weight_unpacked, - local_topk_weights, - local_topk_ids, - self.gate_up_input_scale, - self.gate_up_scales, - self.gate_up_global_scale, - self.down_input_scale, - self.down_scales, - self.down_global_scale, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .gate_up_weight_unpacked = gate_up_weight_unpacked, - .down_weight_unpacked = down_weight_unpacked, - .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), - .gate_up_scales = gate_up.quantization.?.scales, - .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), - .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), - .down_scales = down.quantization.?.scales, - .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - - return fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); -} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig new file mode 100644 index 00000000..92317c7c --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -0,0 +1,355 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const Attributes = contract.Attributes; +const runtime = @import("runtime.zig"); + +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32, + activation: Activation, + enable_pdl: bool, + gemm1_tactic: i32, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32, + + pub fn fromGlobalOptions(opts: GlobalOptions) Options { + return .{ + .workspace_query_device = 0, + .activation = opts.activation, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, + }; + } +}; + +pub fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + +const Inputs = contract.Inputs(zml.Tensor); + +const Outputs = contract.Outputs(zml.Shape); + +fn currentRunners() !*runtime.Runners { + const platform = zml.Compiler.current().platform; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: Activation) i32 { + return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + }; +} + +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; + + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, + ); +} + +fn computeCapability(platform: *const zml.Platform) !u16 { + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: Activation, +) zml.Shape { + const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 2 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 2 or + topk_ids.rank() != 2) + { + return error.InvalidShape; + } + + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + }; + const top_k = topk_ids.dim(.topk); + + if (tokens <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or + !fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = tokens, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, + options, + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + + attributes.runners = @intFromPtr(runners); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const context = runtime.makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + runtime.checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); + + const result = call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const topk_ids = routing.localIds(); + // The runner ignores routes whose local expert id is -1. + const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); + + const runner_options: Options = .fromGlobalOptions(opts); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); + + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); +} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index e5c5b6de..ba3d3c00 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,9 +6,13 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); -const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -17,90 +21,19 @@ test { } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -117,10 +50,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -148,13 +81,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -164,9 +97,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -175,7 +108,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -185,9 +118,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -203,18 +136,18 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { } // TODO(Corentin): unused for now -pub fn supports(opts: FusedExpertsArgs) bool { - if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or - opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { + if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or + down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; - const gate_up = opts.gate_up.weight; - const down = opts.down.weight; + const gate_up = gate_up_.weight; + const down = down_.weight; if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; - const down_scale = opts.down.quantizationScales() orelse return false; - const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); + const gate_up_scale = gate_up_.quantizationScales() orelse return false; + const down_scale = down_.quantizationScales() orelse return false; + const experts = routing.global_num_experts; if (experts < 1 or experts > 512) return false; - var max_width: i64 = opts.hidden_states.dim(.d); + var max_width: i64 = hidden_states.dim(.d); inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { @@ -233,7 +166,7 @@ pub fn supports(opts: FusedExpertsArgs) bool { else => return false, } } - const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); + const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index e129c76c..f835c889 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,21 +1,17 @@ const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_global_scale: ?Tensor = null, - w2_global_scale: ?Tensor = null, -}; - -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -72,57 +68,30 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); - stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); + stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); - - return fusedExpertsImpl( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up_weight_unpacked.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_global_scale = gate_up_global_scale, - .w2_global_scale = down_global_scale, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - gate_up: Tensor, - down: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { .bf16, .f16, .f32 => true, else => false, @@ -130,28 +99,28 @@ pub fn fusedExpertsImpl( stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { + const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { .bf16, .f16, .f32 => true, else => false, }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); - stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); + stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); const d = hidden_states.dim(.d); const num_tokens = hidden_states.dim(.t); - const topk = topk_ids.dim(.topk); + const topk = routing.ids.dim(.topk); const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { @@ -163,29 +132,29 @@ pub fn fusedExpertsImpl( const x_rows = hidden.insertAxes(.d, .{.topk}) .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }).convert(.i32); const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, - opts.w1_scale, + gate_up_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, - ), opts.w1_global_scale, expert_ids); - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); + ), gate_up_global_scale, expert_ids); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, down, - opts.w2_scale, + down_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, - ), opts.w2_global_scale, expert_ids); + ), down_global_scale, expert_ids); - const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 927f95ca..eb697f36 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,14 +4,21 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const tcl = @import("tcl/tcl.zig"); -pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; pub const cute = @import("cute/cute.zig"); -pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); +pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); -pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -20,60 +27,6 @@ test { _ = @import("tests.zig"); } -pub const ProjectionLayout = enum { concatenated, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -/// How a backend expects the expert weights to be stored -/// The experts are loaded in this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - }; -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, -}; - pub const Backend = enum { stablehlo, cute, @@ -85,8 +38,6 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { - // Keep the dtype as an argument because non scheme-specific backends may depend on it later - _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -97,7 +48,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (cutlass_flashinfer.isNvfp4Supported(platform)) + else if (flashinfer_cutlass.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -120,7 +71,13 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, + .furiosa => b: { + const s = scheme orelse break :b .tcl; + break :b switch (s) { + .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, + .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, + }; + }, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -136,7 +93,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -155,13 +112,17 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, + .cute => if (scheme) |s| switch (s) { + .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, + .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } @@ -169,7 +130,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => cutlass_flashinfer.register(platform), + .flashinfer_cutlass => flashinfer_cutlass.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -178,14 +139,6 @@ pub const Backend = enum { } }; -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -196,6 +149,35 @@ pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Te weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); } +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + input: Logical, + ids: Logical, + weights: Logical, + gate_up: Logical, + down: Logical, + }, + out_specs: Logical, + + /// Shards the `.expert` dimension of both projections over `mesh_axis`. + /// Other dimensions, input tokens, routing IDs/weights and output are replicated. + pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { + const experts: Logical = .parse(.{ .expert = mesh_axis }); + return .{ + .in_specs = .{ + .input = .replicated, + .ids = .replicated, + .weights = .replicated, + .gate_up = experts, + .down = experts, + }, + .out_specs = .replicated, + }; + } +}; + /// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. /// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( @@ -205,72 +187,112 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, backend: Backend, + mesh: *const zml.Mesh, + partitioning: Partitioning, opts: Options, ) zml.Tensor { validateRoutingInputs(input, topk_ids, topk_weights); - return switch (backend) { - .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + + const shapes = .{ + .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), + .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), + .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), + .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), + .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), }; -} + const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); -pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} + const tokens = shapes.input.partition(.t); + const experts = shapes.gate_up.partition(.expert); + // Local kernels require complete feature vectors and top-k routing lists. + stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); + // The expert-axis reduction must add contributions for the same token rows. + // Sharding tokens along that axis would mix contributions for different tokens. + stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); + stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); + for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { + stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); + for (0..shape.rank()) |axis| { + stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); + } + } + const expert_axis = experts.meshAxis(); + const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; + if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); -pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { - const x = input.convert(.f32); - return switch (activation) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + const Call = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + opts: Options, + mesh: *const zml.Mesh, + expert_axis: ?u3, + global_experts: u32, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + const routing: LocalRouting = .{ + .ids = args.ids, + .weights = args.weights, + .global_num_experts = args.global_experts, + .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), + .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, }; - - break :b switch (activation) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch + const output = switch (args.backend) { + .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), }; + return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; + } + }; + const args: Call = .{ + .input = input, + .ids = topk_ids, + .weights = topk_weights, + .gate_up = gate_up, + .down = down, + .backend = backend, + .opts = opts, + .mesh = mesh, + .expert_axis = if (parallel) expert_axis else null, + .global_experts = @intCast(gate_up.weight.dim(.expert)), + }; + const Logical = zml.Sharding.Partitioning.Logical; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; + for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + zml.meta.mapAlloc(struct { + fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { + const axis = ctx.axis orelse return .replicated; + if (tensor.count() == 1) return .replicated; + // Packed scales may be untagged; experts are their leading dimension. + const dim = tensor.shape().hasTag(.expert) orelse 0; + stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); + return .{ .mapping = .{ + .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), + .tgt = .init(&.{axis}), + } }; + } + }.spec, zml.Compiler.current().arena.allocator(), .{ + .experts = linear.weight.dim(.expert), + .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, + }, linear, specs) catch @panic("OOM"); + } + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ + .in_specs = .{ + .input = partitioning.in_specs.input, + .ids = partitioning.in_specs.ids, + .weights = partitioning.in_specs.weights, + .gate_up = projection_specs[0], + .down = projection_specs[1], }, - }; + .out_specs = partitioning.out_specs, + }); } diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 09dc360a..bb60594b 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,6 +5,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -12,22 +15,9 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - expert_map: ?Tensor = null, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_bias: ?Tensor = null, - w2_bias: ?Tensor = null, -}; - -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - // TODO(Corentin): find where to put that - //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; - //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -224,120 +214,25 @@ pub fn callGmmEp( } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); + const expert_map_ = routing.expertMap(); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (expert_partition.isSharded()) { - const global_num_experts = down.weight.dim(.expert); - const partial_output = zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: zml.moe.Activation, - global_num_experts: i64, - gate_up_scales: ?zml.Tensor, - bias_gate_up: ?zml.Tensor, - down_scales: ?zml.Tensor, - bias_down: ?zml.Tensor, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_output = fusedExpertsImpl( - self.input, - self.weights_gate_up, - self.weights_down, - self.topk_weights, - self.topk_ids, - .{ - .activation = self.activation, - .global_num_experts = self.global_num_experts, - .expert_map = expert_map, - .w1_scale = self.gate_up_scales, - .w2_scale = self.down_scales, - .w1_bias = self.bias_gate_up, - .w2_bias = self.bias_down, - }, - ); - return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = opts.activation, - .global_num_experts = global_num_experts, - .gate_up_scales = gate_up_scales, - .bias_gate_up = gate_up.bias, - .down_scales = down_scales, - .bias_down = down.bias, - }, - input.shape(), - ); - return zml.ops.allReduce(partial_output, zml.Tensor.add); - } - - return fusedExpertsImpl( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up.weight.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - w1: Tensor, - w2: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { const tokens = hidden_states.dim(.t); const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); - const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); - const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); - const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); + const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); + const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); + const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); + const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); - const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); - if (opts.expert_map) |expert_map| { + const num_experts = routing.global_num_experts; + if (expert_map_) |expert_map| { if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } @@ -345,7 +240,7 @@ pub fn fusedExpertsImpl( const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { + const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); @@ -356,7 +251,7 @@ pub fn fusedExpertsImpl( }; const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (opts.expert_map != null) + const hidden_sorted = if (expert_map_ != null) permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) else blk: { // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route @@ -379,19 +274,19 @@ pub fn fusedExpertsImpl( .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w1_bias) |bias| { + if (gate_up_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); gate_up_out = gate_up_out.add(bias_per_token); } - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w2_bias) |bias| { + if (down_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); down_out = down_out.add(bias_per_token); } diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index e33d2a14..ba723234 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,13 +1,18 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; -pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { +pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -148,23 +153,28 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: zml.moe.Activation, + activation: Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, }; +/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); const tokens = input.dim(.t); const k = topk_ids.dim(.topk); - stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); - const ids = topk_ids.reshape(.{ .route = tokens * k }); + // Nonlocal routes use expert zero for safe gathers; discard their full + // contribution after the down projection, including any projection bias. + const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); + const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); + const ids = safe_ids.reshape(.{ .route = tokens * k }); const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) @@ -176,13 +186,15 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { - return reference(input, ids, weights, gate_up, down, .{ +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { + stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); + return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, .quantize_input = opts.quantize_input, .routing_weight_placement = opts.routing_weight_placement, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 4057eea3..44b56e08 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = ["tcl.zig"], + srcs = [ + "moe.zig", + "tcl.zig", + ], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 4b7bcfb3..f05727ce 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,270 +1,165 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -//! -//! Few tokens (at most `experts / top_k`): activation stationary, each token -//! gathers the weights of its experts and the down projection contracts over -//! the experts and the intermediate axis at once. -//! -//! More tokens: blockwise. The kernel routes on the device: routes are placed -//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a -//! device loop over the used chunks of `BGi` blocks gathers each block's -//! expert weights and runs both projections; outputs are gathered back per -//! route and weighted. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). -/// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); - stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); - stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { + const topk_ids = routing.localIds(); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const gate_up = gate_up_linear.weight; const down = down_linear.weight; - stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); - const t = input.dim(.t); - const k = topk_ids.dim(.topk); - const cfg: Cfg = .{ - .t = t, - .k = k, - .e = gate_up.dim(.expert), - .d = input.dim(.d), - .i = down.dim(.dout), + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); - const x = input.reshape(.{ t, cfg.d }); - const ids = topk_ids.reshape(.{ t, k }); - const weights = topk_weights.convert(.f32).reshape(.{ t, k }); - const out_shape = x.shape(); - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); + const weights = routing.weights.convert(.f32); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; +const Scales = struct { gate_up: Tensor, down: Tensor }; -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.t), - .K = b.axis("K", cfg.k), - .E = b.axis("E", cfg.e), - .D = b.axis("D", cfg.d), - .I = b.axis("I", cfg.i), - .I2 = b.axis("I2", 2 * cfg.i), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, }; -fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { - const t = try b.declareArgs(.{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; } -const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; - -/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times -/// `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; SwiGLU chains several. - const op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); +fn fp8Scales(linear: zml.nn.Linear) Tensor { + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); + const w = linear.weight; + const scale = q.scales; + stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); + return scale; } -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// IO-bound tactics for activation stationary, compute-bound for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, blockwise: bool) void { - if (blockwise) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - .propagate_sparse_axis_from_op = "None", - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - .propagate_sparse_axis_from_op = "None", - }); +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; + +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); + } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); } -pub const ActivationStationary = tcl.Kernel(Cfg, .{ - .name = "moe_activation_stationary", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; - var op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); - // Routing weights scale the activations, so the down projection - // also sums over the experts. - const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - }.run, -}); + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; -/// Rows per expert block: the next power of two of the routes per expert. -pub fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); } -const blocks_per_chunk = 32; +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; +} -pub const Blockwise = tcl.Kernel(Cfg, .{ - .name = "moe_blockwise", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); - const a: Axes = .init(b, cfg); - const in = try args(b, a); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); - const act = try gated(b, a, h, &.{ BGi, BB }, null); - op = b.tensorOperation(.{}); - const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); - } - }.run, -}); +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; +} test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; @@ -276,15 +171,15 @@ test "tcl fused experts run on furiosa" { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { - return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ - .activation = .{ .swiglu = .{} }, + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { + return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ + .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -336,7 +231,12 @@ test "tcl fused experts run on furiosa" { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - act[n] = g / (1 + @exp(-g)) * u; + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; } for (0..d) |m| { var y: f32 = 0; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index 2bdebdfe..c74406c3 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,12 +5,15 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); -const Placement = @import("moe.zig").RoutingWeightPlacement; const Case = struct { backend: zml.moe.Backend, - activation: zml.moe.Activation = .{ .swiglu = .{} }, + activation: Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -22,15 +25,22 @@ const Case = struct { width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, + expert_mesh: *const zml.Mesh, + + fn inputShape(c: Case) Shape { + const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + return shape.withPartitioning(c.expert_mesh, spec); + } pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); + const x: Tensor = .fromShape(c.inputShape()); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const x: Tensor = .fromShape(c.inputShape()); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -52,6 +62,7 @@ const Case = struct { var output: zml.Bufferized(Outputs) = undefined; runner.run(io, .{input}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Outputs, &output); + try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. @@ -168,20 +179,16 @@ fn makeExpertFixture(c: Case) ExpertFixture { return fixture; } -/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. -fn interleave(tensor: Tensor) Tensor { - const tags = tensor.shape().tags(); - return tensor.withTags(.{ .expert, .out, .in }) - .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) - .transpose(.{ .expert, .mid, .projection, .in }) - .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); +fn interleave(tensor: Tensor, rows: i64) Tensor { + return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) + .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); } /// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> /// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) - .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); + return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); } fn packFp4Bytes(weight: Tensor) Tensor { @@ -189,10 +196,11 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, - .interleaved => interleave(tensor), + .row_interleaved_1 => interleave(tensor, 1), + .row_interleaved_8 => interleave(tensor, 8), }; } @@ -210,17 +218,29 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF } for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; + switch (layout.packing) { + .swizzled_scales, .flashinfer_nvfp4 => { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + }, + .mxfp4_interleaved8 => { + const q = &projection.quantization.?; + const sf = q.scales; + q.scales = sf.bitCast(.u8) + .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) + .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) + .withTags(sf.shape().transpose(.{ 0, 2, 1 })); + }, + .plain => {}, } switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); const q = &projection.quantization.?; - q.scales = q.scales.bitCast(.u8); if (scheme == .nvfp4) { + q.scales = q.scales.bitCast(.u8); // CuTe takes the input quantization multiplier and a GEMM // output multiplier that folds in its inverse. const input_scale = q.input_scale.?.asMultiplier(); @@ -263,14 +283,26 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: zml.moe.Options = .{ + const options: GlobalOptions = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, }; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); - const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ + const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ + .in_specs = .{ + .input = data_spec, + .ids = data_spec, + .weights = data_spec, + .gate_up = expert_spec, + .down = expert_spec, + }, + .out_specs = data_spec, + }, options); + // Keep the oracle input replicated so token sharding does not change its GEMM rounding. + const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -285,7 +317,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]zml.moe.Activation{ +const gated_activations = [_]Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -295,8 +327,8 @@ const gated_activations = [_]zml.moe.Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]zml.moe.Activation{ +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -334,7 +366,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const zml.moe.Activation, + activations: []const Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, @@ -346,7 +378,7 @@ const Matrix = struct { } /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { std.debug.assert(index < self.count()); var remaining = index; const shape = self.shapes[remaining % self.shapes.len]; @@ -374,6 +406,7 @@ const Matrix = struct { .width = shape.width, .intermediate = shape.intermediate, .experts = shape.experts, + .expert_mesh = expert_mesh, }; } }; @@ -414,7 +447,7 @@ const Pipeline = struct { var index: usize = 0; for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, local_index); + const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); if (available) { case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { error.Closed => {}, @@ -591,6 +624,7 @@ test "Triton MoE prefill launch buckets fit shared memory" { for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { const case: Case = .{ .backend = .triton, + .expert_mesh = platform.replicated_mesh, .tokens = tokens, .width = 256, .intermediate = 128, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 3eb457da..6e4006fa 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,6 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1026,7 +1027,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1092,7 +1093,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 908cd163..8ba799a5 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,10 +6,14 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const kernels = @import("moe.zig"); const toDType = zml.kernel.triton.from; -const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -35,109 +39,38 @@ test { pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. if (gate_up.quantization) |q| switch (q.scheme) { .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); + return fusedExpertsMxfp4(input, routing, gate_up, down, options); }, else => {}, }; - return fusedExpertsBase(input, ids, weights, gate_up, down, options); + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + return fusedExpertsBase(input, routing, gate_up, down, layout, options); } pub fn fusedExpertsBase( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + layout: ProjectionLayout, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -154,10 +87,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -185,13 +118,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -201,9 +134,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -212,7 +145,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -222,9 +155,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -561,14 +494,14 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { - return zml.moe.applyActivation(x, activation, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); @@ -577,7 +510,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -733,7 +666,8 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. @@ -765,13 +699,12 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsImpl(.{ - .hidden_states = x, - .gate_up = gate_up, - .down = down, - .topk_ids = route.addConstant(2), - .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), - .gate_up_layout = layout, + return fusedExpertsBase(x, .{ + .ids = route.addConstant(2), + .weights = route.convert(.f32).addConstant(1).scale(0.25), + .global_num_experts = 8, + .local_num_experts = 8, + }, gate_up, down, layout, .{ .routing_weight_placement = placement, .quantize_input = false, .activation = .{ .swiglu = .{} }, @@ -786,7 +719,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); @@ -866,58 +799,13 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); } -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: zml.moe.Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - - const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); - - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .w1 = gate_up.weight.bitCast(.u8), - .s1 = gq.scales, - .w2 = down.weight.bitCast(.u8), - .s2 = dq.scales, - .global_experts = gate_up.weight.dim(.expert), - .topk = ids.dim(.topk), - .limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - .expert_parallel = expert_parallelism, - }; - - return if (expert_parallelism) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); -} - pub const Config = struct { tokens: i64, hidden: i64, @@ -952,210 +840,233 @@ pub const Config = struct { } }; -pub fn validateInputs( +fn validateInputs( c: Config, input: zml.Tensor, ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, -) !void { - if (s1.rank() != 3 or s2.rank() != 3) return error.InvalidInputShape; - if (s1.dim(2) < @divTrunc(c.hidden, 32) or - s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; + expert_weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, +) void { + stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); + const shapes = .{ - .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), - .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, s1.dim(2) }, .f8e8m0), - .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), - .s2 = zml.Shape.init(.{ c.experts, c.hidden, s2.dim(2) }, .f8e8m0), + .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), + .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), + .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), + .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), }; - if (!input.shape().eql(shapes.x)) return error.InvalidInputShape; - if (!ids.shape().eql(shapes.ids)) return error.InvalidInputShape; - if (!weights.shape().eql(shapes.scales)) return error.InvalidInputShape; - if (!w1.shape().eql(shapes.w1)) return error.InvalidInputShape; - if (!s1.shape().eql(shapes.s1)) return error.InvalidInputShape; - if (!w2.shape().eql(shapes.w2)) return error.InvalidInputShape; - if (!s2.shape().eql(shapes.s2)) return error.InvalidInputShape; + stdx.debug.assert(input.shape().eql(shapes.input) and + ids.shape().eql(shapes.ids) and + expert_weights.shape().eql(shapes.expert_weights) and + // Note: we accept both i8 and u8 for gate_up and down + // TODO: we should instead use f4e2m1 + (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and + gate_up_scales.shape().eql(shapes.gate_up_scales) and + (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and + down_scales.shape().eql(shapes.down_scales), + \\Invalid MXFP4 tensor layout: + \\ - expected input={f}, got {f} + \\ - expected ids={f}, got {f} + \\ - expected expert_weights={f}, got {f} + \\ - expected gate_up={f}, got {f} + \\ - expected gate_up_scales={f}, got {f} + \\ - expected down={f}, got {f} + \\ - expected down_scales={f}, got {f} + , .{ + shapes.input, input, // + shapes.ids, ids, // + shapes.expert_weights, expert_weights, // + shapes.gate_up, gate_up, // + shapes.gate_up_scales, gate_up_scales, // + shapes.down, down, // + shapes.down_scales, down_scales, // + }); + + stdx.debug.assert( + gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and + down_scales.dim(2) >= @divTrunc(c.intermediate, 32), + "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", + .{ c, gate_up_scales, down_scales }, + ); } -const Context = struct { +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, - global_experts: i64, - topk: i64, - limit: f32, - routing_weight_placement: RoutingWeightPlacement, - expert_parallel: bool, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; - var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); - const tokens = ids.dim(.token); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - ids = ids.sub(partition_id.scale(experts)); - } - const x = self.input.reshape(.{ tokens, hidden }); - const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = self.global_experts, - .topk = self.topk, - .swiglu_limit = self.limit, - .routing_weight_placement = self.routing_weight_placement, - }; + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, self.w1, self.s1, self.w2, self.s2) catch @panic("Invalid MXFP4 tensor layout"); + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); - const bn = 16; - const bk = 512; + const x = input.reshape(.{ tokens, hidden }); + const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = routing.global_num_experts, + .topk = topk, + .swiglu_limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + }; - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, .num_warps = 4, .num_stages = 1, }, ); - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - map = s.map; - perm = s.perm; - sched = s.sched; - } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = self.w1, .ws = self.s1.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s1.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = self.w2, .ws = self.s2.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s2.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - const result = kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; + map = s.map; + perm = s.perm; + sched = s.sched; } -}; + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = gq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = dq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + return kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(input.shape().dims()).withTags(input.shape()); +} diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 9fd6a355..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -54,7 +54,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -115,6 +115,31 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { + const Call = struct { + inputs: Inputs, + case: cases.Case, + + fn local(self: @This(), _: zml.Shape) zml.Tensor { + return fixtures.forward(self.inputs, self.case); + } + }; + const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + .in_specs = .{ .inputs = .{ + .q = heads, + .k = heads, + .v = heads, + .table = .replicated, + .lengths = .replicated, + .starts = .replicated, + .sinks = .parse(.{ ._0 = .model }), + .indices = .replicated, + } }, + .out_specs = heads, + }); +} + fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index e572cce1..ef78cc49 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,10 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const paged = @import("../paged_attention.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); -const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -64,7 +64,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); @@ -72,57 +72,44 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages = k_.dim(.page), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), + .sink = opts.sink != null, + .indexed = opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const indices = switch (self.opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => zml.Tensor.zeroes(.init(.{ q.dim(.b), 1 }, .i32)), - }; - const cfg: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages = k.dim(.page), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .dtype = zml.kernel.cute.from(q.dtype()), - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), - .sink = self.opts.sink != null, - .indexed = self.opts.mask == .indices, - .topk = indices.dim(-1), - }; - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - return kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, - .block = .{ 32, 1, 1 }, - }).out.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 8c289b04..c0815d1d 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,5 +1,7 @@ const std = @import("std"); + const zml = @import("zml"); + const cases = @import("cases.zig"); pub const Fixture = struct { @@ -122,7 +124,7 @@ pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { }, }, }; - return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); } pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 89761bdd..12d6273c 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -317,7 +317,7 @@ fn produce(t: Tile) void { const id = tiles.iv.mul(c.features() * 2).add(stage); var reuse = b.openIf(id.ge(c.buffers())); { - b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); reuse.close(); } const source = if (stage < c.features()) t.k else t.v; @@ -339,7 +339,7 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { { for (0..@as(usize, @intCast(c.features()))) |part| { const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); - b.mbarrierWait(t.ready(id), t.phase(id)); + b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); b.tcgen05Fence(.after); const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); @@ -353,22 +353,22 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); } - b.tcgen05Commit(t.released(id)); + b.tcgen05Commit(t.released(id), .{}); } - b.tcgen05Commit(t.shared.add(c.completion())); + b.tcgen05Commit(t.shared.add(c.completion()), .{}); leader.close(); } - b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1)); + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); b.tcgen05Fence(.after); // A fast warp must not wrap completion parity before every consumer has // observed the preceding PV completion. - if (!qk) b.barrierSync(1, 128); + if (!qk) b.barrierCtaSyncAligned(1, 128); } fn publishProbabilities(t: Tile) void { t.b.fenceProxyAsyncShared(); t.b.tcgen05Fence(.before); - t.b.barrierSync(1, 128); + t.b.barrierCtaSyncAligned(1, 128); t.b.tcgen05Fence(.after); } @@ -393,7 +393,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu leader.close(); } } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); var result: [heads]V = undefined; for (0..heads) |h| { var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); @@ -403,7 +403,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu } result[h] = value; } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); return result; } @@ -514,7 +514,7 @@ fn prefill(t: Tile) void { preserve.yieldElse(@as([64]V, @splat(zero))); } @memcpy(&saved, &preserve.results); - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); } multiply(t, tiles.iv, true); var scores: [64]V = undefined; diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index 664720ee..afc4298e 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,7 +2,6 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -19,61 +18,48 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .decode = decode, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; - var c: kernel.Config = .{ - .architecture = if (cc.minor >= 3) .sm103 else .sm100, - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .decode = decode, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); - // An unused scalar stays a constant buffer; broadcasting a dummy for - // every head would add a helper kernel to each attention invocation. - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = p.block_table, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ - .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; - const ArchitectureCheck = struct { fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6dfeb7e2..6645abf1 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,7 +1,6 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,66 +17,53 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - var c: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .warps_per_query = if (decode) 4 else 2, - .kv_tile = if (q.dim(.hd) == 512) 32 else 64, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - // Mixed batches also contain single-token rows. Partition their KV - // range so one long decode cannot hold up all the prefill CTAs. - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); - const out = kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, - .block = .{ 128, 1, 1 }, - }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 5624de0e..8ccc6fd9 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -10,11 +10,13 @@ transposes queries and caches into the kernels' layouts, launches the the output. Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` -only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head -dimension 512 past 4096 KV positions. Everything else, `f16` decode included, -runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, -pages of 8, 16 or 32 positions), then `zero_padding_rows`. +decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which +is `bf16` only: a threadgroup holds up to four query heads of a KV head, times +splits of the KV positions, at most 256 threads. Rows split over the positions +are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count +and the number of rows. Everything else, `f16` decode included, runs +`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of +8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale are passed to the kernels. Explicit key indices are rejected, and diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e2d68398..2cec68f5 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,8 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); - -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); @@ -86,8 +85,8 @@ const DispatchOptions = struct { scale: f32, }; -/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at -/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. +/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled +/// otherwise. /// `q` and the result are `[tokens, heads, head_dim]`, the caches /// `[pages, block_size, kv_heads, head_dim]`, `block_table` /// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` @@ -106,7 +105,7 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table const max_blocks: i32 = @intCast(block_table.dim(1)); const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 128 or head_dim == 256 or head_dim == 512) and + (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and dtype == .bf16; if (vec_decode) { @@ -129,35 +128,30 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table }; const kv_capacity = max_blocks * block_size; - if (head_dim == 512 and kv_capacity > 4096) { - const nwg = 4; - const nrows = total_q_tokens * num_heads; + const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); + const nrows = total_q_tokens * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); - const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, - .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, 8, 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), - .constants = .{ .args = .of(args) }, - }).dst; + const cfg: paged_attn.FaVecPaged.Cfg = .{ + .head_dim = head_dim, + .block_size = block_size, + .position_stride = num_kv_heads * head_dim, + .heads = launch.heads, + .splits = launch.splits, + }; + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ - .cfg = .{ .nwg = nwg, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), 1, 1 }, - .threadgroup = .{ 32, nwg, 1 }, - .constants = .{ .args = .of(@as(i32, nrows)) }, - }).dst; - } - - var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; - if (head_dim == 512 and idx > 1) idx = 1; - const nsg = ([_]i32{ 4, 8, 16 })[idx]; - - return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, - .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(nsg), 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, .constants = .{ .args = .of(args) }, }).dst; } @@ -211,3 +205,36 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 20429565..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,9 +3,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; @@ -18,25 +17,18 @@ const masked_score = -flt_max / 2.0; const max_half = 65504.0; const simd_width = 32; -fn pad2(x: i32, n: i32) i32 { - return (x + n - 1) & ~(n - 1); -} - -/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup -/// of `nsg` simdgroups per (position split, query head, query row). +/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. pub const FaVecPaged = struct { pub const Cfg = struct { - /// The key and value head dimension, DK = DV. + /// The key and value head dimension. head_dim: i32, - /// Simdgroups splitting the KV positions. - nsg: i32, /// The page size. block_size: i32, /// The in-page position stride, n_kv * head_dim elements. position_stride: i32, - /// Position-split threadgroups. Above 1 the kernel writes f32 partials - /// for FaVecPagedReduce. - nwg: i32, + heads: i32, + splits: i32, }; pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ @@ -66,17 +58,10 @@ pub const FaVecPaged = struct { .run = run, }); - /// The threadgroup memory of a launch with `nsg` simdgroups. - pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { - const padded = pad2(head_dim, 128); - const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; - return @intCast(@divFloor(bytes + 15, 16) * 16); - } - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { switch (cfg.head_dim) { - 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), } const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; @@ -89,158 +74,90 @@ pub const FaVecPaged = struct { .seq_lens = int_in, .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .shmem_f16 = .{ .threadgroup = .half }, .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, .tgpg = .{ .threadgroups_per_grid = .uint3 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, }); impl(b, cfg, a); } - /// The attention body. fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const DK = cfg.head_dim; - const DV = cfg.head_dim; - const NE = 1; - const C = 32; - const NSG = cfg.nsg; + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; const BS = cfg.block_size; const PS = cfg.position_stride; - const NWG = cfg.nwg; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + const num_heads = a.args.field("num_heads"); + const group = num_heads.div(a.args.field("n_kv")); + const groups = group.div(HG); const r = a.tgpig.at(2).to(.int); - const iq2 = a.tgpig.at(1).to(.ushort); - const iwg = a.tgpig.at(0).to(.short); - - const DK4 = @divExact(DK, 4); - const DV4 = @divExact(DV, 4); - - const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + // Padding rows: FaVecPagedReduce zeroes them when there are splits. var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - const zero = b.constant(.float4, 0).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), zero); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); - lane0.close(); - } + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); + var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); + { + dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); + loop.close(); } - first.close(); } b.return_(); padding.close(); } - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + const seq = sequenceOf(b, a, r); + const kv_len = seq.kv_len; + const kv_lo = seq.kv_lo; + const ic0_lo = kv_lo.div(C); + const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); { - first_seq.assign(mid); - half.elseBranch(); + b.return_(); + idle.close(); } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); } - const seq_idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); - const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); - const kv_total = a.seq_lens.load(seq_idx); - const kv_len = kv_total.sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); + const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const PK = pad2(DK, 128); - const PK4 = @divExact(PK, 4); - const PV = pad2(DV, 128); - const PV4 = @divExact(PV, 4); - const NW = simd_width; - const NL = @divExact(NW, NE); - const SH = 4 * C; - - const shmem = a.shmem_f16; - const sq4 = shmem.add(0 * PK).to(.half4); - const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); - const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); - const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); - const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); - // This lane's output, until the simdgroups merge theirs. - const lane_so4 = so4.add(a.tiisg); - - const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); - const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); - const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - - const q4 = q.to(.bfloat4); - var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const i = load_q.iv; - var value = b.openIfElse(i.lt(DK4), .{.half4}); - { - value.yieldThen(.{q4.load(i).to(.half4)}); - } - { - value.yieldElse(.{b.constant(.half4, 0)}); - } - sq4.store(i, value.results[0]); - load_q.close(); - } - var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); - { - lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); - zero_o.close(); - } - var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); - { - ss4.store(zero_s.iv, b.constant(.float4, 0)); - zero_s.close(); + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); } - b.threadgroupBarrier(.mem_threadgroup); - - // When DV4/NL is a multiple of NW every lane updates the output, so the - // `ty == 0` guard is dropped at comptime. - const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; - const full: metal.LoopOpts = .{ .unroll = .full }; const S = b.local(.float, 0.0); const M = b.local(.float, -flt_max / 2.0); - const tx = a.tiisg.rem(NL).to(.short); - const ty = a.tiisg.div(NL).to(.short); - - const ic0_lo = kv_lo.div(C); - const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); + const ic0 = b.local(.int, ic0_lo.add(iwg)); var blocks = b.openLoop(.{}); { const ic = ic0.mul(C); @@ -250,175 +167,103 @@ pub const FaVecPaged = struct { past.close(); } - const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); - sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); - - const pq4 = sq4.add(tx); - const mqk = b.zeroedArray(.float, C / NE); - var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = qk.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dots.iv; - const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); - mqk.store(cc, mqk.load(cc).add(dot)); + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); dots.close(); } - mqk.store(cc, b.simdSum(mqk.load(cc))); + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } qk.close(); } - const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); - ss.store(b.lift(NE).mul(tx).add(ty), score); - - b.simdgroupBarrier(.mem_threadgroup); + const pos = ic.add(a.tiisg.to(.int)); + const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); + const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); const m = M.rvalue(); - const s = ss.load(a.tiisg); - - M.assign(b.simdMax(b.max(M.rvalue(), s))); - + M.assign(b.simdMax(b.max(M.rvalue(), score))); const ms = b.exp(m.sub(M)); - const vs = b.exp(s.sub(M)); - + const vs = b.exp(score.sub(M)); S.assign(S.mul(ms).add(b.simdSum(vs))); - ss.store(a.tiisg, vs); - - var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const at = rescale.iv.mul(NL); - lane_so4.store(at, lane_so4.load(at).mul(ms)); + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); rescale.close(); } - if (rescale_lanes) |*l| l.close(); - b.simdgroupBarrier(.mem_threadgroup); - - const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); - var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - lo.store(zero_lo.iv, b.constant(.float4, 0)); - zero_lo.close(); - } - - const sst = ss.add(ty); - - var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = pv.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dims.iv; - const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); - lo.store(ii, lo.load(ii).add(weighted)); + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); dims.close(); } pv.close(); } - var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - const ii = add_o.iv; - const partial = lo.load(ii); - lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); - add_o.close(); - } - if (add_lanes) |*l| l.close(); blocks.next(); - ic0.assign(ic0.add(NWG * NSG)); + ic0.assign(ic0.add(live)); blocks.close(); } - var stats = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S); - ss.store(1, M); - stats.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const rr = b.local(.short, @divFloor(NSG, 2)); - var halving = b.openLoop(.{}); - { - halving.body(rr.gt(0)); - } - { - var group = b.openIf(a.sgitg.lt(rr)); + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const S0 = ss.load(0); - const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); - - const M0 = ss.load(1); - const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); - - const M_ = b.max(M0, M1); - - const ms0 = b.exp(M0.sub(M_)); - const ms1 = b.exp(M1.sub(M_)); - - const S_ = S0.mul(ms0).add(S1.mul(ms1)); - - var lane0 = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S_); - ss.store(1, M_); - lane0.close(); - } - - var merge = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const i = merge.iv; - so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); - merge.close(); - } - group.close(); + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); } - b.threadgroupBarrier(.mem_threadgroup); - rr.assign(rr.shr(1)); - halving.close(); } - - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), out); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const part = so4.load(loop.iv); - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const sum = ss.load(0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); - const max = ss.load(1); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); - lane0.close(); - } + var first = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); } - first.close(); } + if (first) |*scope| scope.close(); } /// The address of position `p`'s bf16 row in `cache`, through the page @@ -430,85 +275,144 @@ pub const FaVecPaged = struct { } }; -/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, -/// one threadgroup per row and head. +/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). +fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(idx)); + const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); + const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; +} + +fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { + const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); + return b.min(splits, chunks); +} + +/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. pub const FaVecPagedReduce = struct { pub const Cfg = struct { - /// The partials to merge, FaVecPaged's `nwg`. - nwg: i32, + splits: i32, /// The head dimension. head_dim: i32, }; - pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ - .{ .name = "nrows", .type = .int }, - } }; + pub const simdgroups = 8; pub const Kernel = metal.Kernel(Cfg, .{ .name = "fa_vec_paged_reduce", - .inputs = &.{"htmp"}, + .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, .outputs = &.{"dst"}, .constants = &.{"args"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, + .args = .{ .constant_struct = FaVecPaged.Args }, .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .seq_lens = int_in, + .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, }); - const nwg = cfg.nwg; + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); const DV = cfg.head_dim; - const rid = a.tgpig.to(.ulong); - const iwg = a.tiisg.to(.short); - const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; - var max = b.openIfElse(iwg.lt(nwg), .{.float}); - { - max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); - } - { - max.yieldElse(.{-flt_max / 2.0}); - } - const M = max.results[0]; - var sum = b.openIfElse(iwg.lt(nwg), .{.float}); - { - sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); - } - { - sum.yieldElse(.{0.0}); - } - - const m = b.simdMax(M); - const ms = b.exp(M.sub(m)); - const S = b.simdSum(sum.results[0].mul(ms)); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const DV4 = @divTrunc(DV, 4); - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); + + const live = b.local(.int, 0); + var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - const i = loop.iv; - var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); + live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); + real.close(); + } + var empty = b.openIf(live.le(0)); + { + var lane0 = b.openIf(a.tiisg.eq(0)); { - part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); - } - { - part_.yieldElse(.{b.constant(.float4, 0)}); - } - const part = part_.results[0]; - const v = b.simdSum(part); - var lane0 = b.openIf(iwg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); lane0.close(); } - loop.close(); + b.return_(); + empty.close(); + } + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); } } }; @@ -984,13 +888,14 @@ pub const PagedAttentionTiled = struct { test "the paged attention kernels emit verified modules" { const allocator = std.testing.allocator; - for ([_]i32{ 128, 256, 512 }) |head_dim| { - for ([_]i32{ 4, 8, 16 }) |nsg| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); } } - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); for ([_]Type{ .half, .bfloat }) |dtype| { allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index ef9e852c..84716379 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -153,32 +153,28 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", .{ q.dim(.hd), target_head_dim }, ); - var q_padded = q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); - q_padded._shape._sharding = q._shape._sharding; - return q_padded; + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const kv_pages_partitioned = kv_pages.withPartitioning(mesh, .{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(mesh, .{ .h = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); - const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); stdx.debug.assert( cache_head_dim == alignedKernelHeadDim(logical_head_dim), "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", .{ cache_head_dim, logical_head_dim }, ); - const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(mesh, .{ .b = .replicated }); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); + const query_start_len = parameters.query_start_len; return .{ .q = q_ragged, - .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(mesh, .{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(mesh, .{ ._0 = .replicated, ._1 = .replicated }), + .kv_pages = kv_pages, + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(mesh, .{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len), }; } @@ -224,57 +220,9 @@ pub const mosaic_tpu = struct { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); - const prepared = prepareInputs(parameters, q, kv_cache); - - const q_out = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - kv_pages: zml.Tensor, - seq_lens: zml.Tensor, - block_table: zml.Tensor, - query_start_len: zml.Tensor, - num_seqs: zml.Tensor, - opts: AttentionOptions, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const prepared_inputs: PreparedInputs = .{ - .q = self.q, - .kv_pages = self.kv_pages, - .seq_lens = self.seq_lens, - .block_table = self.block_table, - .query_start_len = self.query_start_len, - .num_seqs = self.num_seqs, - }; - - const cfg = buildCfg(prepared_inputs, self.parameters, self.opts); - const q_out = raggedPagedKernelCall( - prepared_inputs.q, - prepared_inputs.kv_pages, - prepared_inputs.seq_lens, - prepared_inputs.block_table, - prepared_inputs.query_start_len, - prepared_inputs.num_seqs, - cfg, - ); - stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); - return q_out; - } - }).body, - .{ - .q = prepared.q, - .kv_pages = prepared.kv_pages, - .seq_lens = prepared.seq_lens, - .block_table = prepared.block_table, - .query_start_len = prepared.query_start_len, - .num_seqs = prepared.num_seqs, - .opts = opts, - .parameters = parameters, - }, - prepared.q.shape(), - ); - + const cfg = buildCfg(prepared, parameters, opts); + const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); const restored = restoreQueryHeads(q, q_out); stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); return restored; diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 99065140..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,12 +3,14 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); -const cute = @import("cute/cute.zig"); +pub const AttentionMask = @import("options.zig").AttentionMask; +pub const AttentionOptions = @import("options.zig").AttentionOptions; +pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const triton = @import("triton/triton.zig"); -pub const tcl = @import("tcl/tcl.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const triton = @import("triton/triton.zig"); const PagedAttention = @This(); @@ -30,7 +32,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { - .cuda => .triton, + .cuda => if (cute.isAvailable(platform)) .cute else .triton, .rocm => .triton, .oneapi => .triton, .tpu => .mosaic_tpu, @@ -151,41 +153,6 @@ pub const Parameters = union(Backend) { } }; -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: zml.Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -232,49 +199,53 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } +}; - pub fn getPage( - self: KvCache, - page_index: zml.Tensor, - ) [2]zml.Tensor { - return switch (self) { - .split => |split| .{ - split.k.slice(split.k.axis(.page), .dynSingle(page_index)), - split.v.slice(split.k.axis(.page), .dynSingle(page_index)), +pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + _ = k; + _ = v; + const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); +} + +const Attention = struct { + parameters: Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .cute => |p| switch (args.kv_cache) { + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (args.kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + }, + .metal => |metal_parameters| switch (args.kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .tcl => |params| switch (args.kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), }, - .dense => @panic("TODO"), }; } }; -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { - _ = k; - _ = v; - return switch (parameters) { - .cute => |p| switch (kv_cache) { - .split => |split| cute.pagedAttention(p, q, split.k, split.v, opts), - .dense => @panic("CuTe paged attention requires split KV pages"), - }, - .triton => |triton_parameters| switch (kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), - }, - .metal => |metal_parameters| switch (kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), - .tcl => |params| switch (kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, q, split.k, split.v, opts), - .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), - }, - }; -} - test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ .arena = undefined, @@ -335,21 +306,22 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; - const model_meshe = platform.meshes.get("model").?; + const tp = zml.testing.tpMesh(); + const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), .kv_cache = .{ .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), }, }, }; - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); @@ -462,11 +434,22 @@ test pagedAttention { backend_options_args.backend = backend; const parameters: Parameters = .init(.fromBackend(backend_options_args)); + const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const partitioning: Partitioning = .{ + .in_specs = .{ + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), + .q = head_spec, + .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, + .opts = .{ .mask = .{ .standard = {} }, .sink = null }, + }, + .out_specs = head_spec, + }; + const exe = try platform.compileFn( allocator, io, pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index ab0ffee2..55ee01d5 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,12 @@ const zml = @import("../../../zml.zig"); -const KvCache = @import("../paged_attention.zig").KvCache; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,11 +14,12 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = kv_cache.split.k.dim(.k_chunk); + const page_size = k.dim(.k_chunk); + const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -28,16 +29,17 @@ pub fn pagedAttention( // Note: we do all partial softmax in f32 .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(q.shape()), + .out = .zeroes(padded_q.shape()), }, ); - return final_state.out; + return final_state.out.slice(.b, .{ .end = q.dim(.b) }); } const AttentionLoop = struct { q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -65,7 +67,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -93,7 +95,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } /// Decode version: num_slots == 1, we multiply one query with a page of keys. @@ -112,7 +114,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } }, .{ @@ -133,7 +135,10 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { @@ -162,7 +167,11 @@ const AttentionLoop = struct { return og_softmax.merge(partial); } - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { + fn nextQueryStart(self: While, state: State) zml.Tensor { + return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { return zml.ops.if2( query_has_more_keys, // If the query has more pages: update partial softmax with attention on current page, delay updating output. @@ -181,7 +190,7 @@ const AttentionLoop = struct { .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = state.slot_id.addConstant(num_slots), + .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index ce8bdb4f..d04b684e 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1,42 +1,55 @@ -//! Furiosa RNGD paged attention in TCL, using the common/flash attention -//! algorithms from Furiosa's SDK (`furiosa-kernels 2026.4.0` reference). -//! This implementation retains FP32 normalization/accumulation and BF16 -//! contractions. The exact common SDK operation materializes scores in BF16; -//! it fails the required unit-scale tolerance and is not a selectable backend. -//! -//! Queries are cut in blocks of rows of one sequence (one token per block for -//! decode). A context of at most `chunk_keys` keys runs Furiosa's one-shot -//! attention: gather the keys by token slot, contract, mask, one softmax, -//! contract. Longer contexts run Furiosa's flash attention: a device loop over -//! chunks of `chunk_keys` keys up to the longest sequence, gathering whole -//! pages and accumulating the softmax statistics in f32. -//! -//! Decode without sinks or windows, up to `decode_keys` keys, instead runs one -//! fusible kernel per layer over the whole pages each token sees, with the -//! token metadata every layer shares computed once per forward pass. +//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. +//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. +//! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); -const stdx = @import("stdx"); const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const paged_attention = @import("../paged_attention.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); +/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to +/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. +/// Remove once 8192-key chunks compile. const chunk_keys = 4096; +const flash_chunk_pairs = 512 * 2048; +const prefill_chunk_keys = 2048; const prefill_block = 32; +/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout +/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. +/// Remove once any count compiles. +fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { + const blocks = @divTrunc(tokens + sequences * (block - 1), block); + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); +} + +fn prefillBlock(tokens: i64, sequences: i64) i64 { + const short = prefill_block / 2; + return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; +} + +/// A straight-line head over each token's first `head` keys, then a device loop over +/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. +pub const Plan = struct { + head: i64, + chunk: i64, + group: i64, +}; + +pub var calibration_plan: ?Plan = null; + pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches - /// `{.page, .k_chunk, .hkv, .hd}`. Sinks are one logit per query head. - /// Unwritten cache entries are excluded before the contractions; callers - /// need only populate the tokens covered by their sequence lengths. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. + /// Sinks are one logit per query head. + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -46,7 +59,7 @@ pub const paged = struct { stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); const compiler = zml.Compiler.current(); - stdx.debug.assert(compiler.platform.target == .furiosa and compiler.meshes[0].numDevices() == 1, "TCL attention requires one Furiosa chip", .{}); + stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); defer attributes.deinit(); @@ -54,25 +67,33 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + return attend(parameters, q, k_cache, v_cache, opts); + } + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); - // TCC 2026.5 has no DPE tactic for 32-key attention. Pad to - // at least 64 keys; the extra pages remain invalid and contribute zero. + // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to + // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. + // Remove once 16- to 48-key decode compiles. const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; - // Chunks hold whole pages and a multiple of 64 keys: TCC 2026.5 fails - // to allocate a chunk of 4080 keys (85 pages of 48). + // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 + // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: + // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; const alignment = if (key_alignment <= chunk_keys) key_alignment else page; - const chunk = if (keys <= chunk_keys) keys else @min(keys, @max(1, @divTrunc(chunk_keys, alignment)) * alignment); const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); - const blocks = (std.math.divCeil(i64, q.dim(.b), prefill_block) catch unreachable) + parameters.seq_lens.dim(.b); - // Regular block capacities limit layout-search cost for the fused graph. - // Extra blocks retain zero valid lengths and invalid query positions. - const cfg: Cfg = .{ + const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); + const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); + const rows = block_count * block; + // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. + const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; + const chunk = @min(keys, chunk_limit); + var cfg: Cfg = .{ .tokens = q.dim(.b), .sequences = parameters.seq_lens.dim(.b), - .b = if (decode) q.dim(.b) else @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable), - .i = if (decode) 1 else prefill_block, + .b = block_count, + .i = block, .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, .chunk = chunk, .hkv = q.dim(.hkv), @@ -86,10 +107,24 @@ pub const paged = struct { .is_causal = opts.mask.isCausal(), .window = @intCast(opts.mask.slidingWindow() orelse 0), }; - if (decode and !cfg.use_sink and cfg.window == 0 and keys <= decode_keys) { - return decodeAttention(parameters, q, k_cache, v_cache, .{ + const plain_decode = decode and !cfg.use_sink and cfg.window == 0; + // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens + // over a non-power-of-two page count fail to compile, so straight decode reads + // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, + // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. + const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; + const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; + const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); + if (plan.head > 0 and plan.chunk > 0) { + stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); + return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); + } + if (plan.chunk == 0 and straight) { + return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ .t = cfg.tokens, - .a = keys, + .a = straight_keys, + .group = plan.group, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, @@ -97,34 +132,90 @@ pub const paged = struct { .pg = page, .scale = cfg.scale, .is_causal = cfg.is_causal, + .window = cfg.window, + .use_sink = cfg.use_sink, }).convert(q.dtype()); } + cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; + cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size + // other than a power of two fails ("Inconsistent tensor group + // offsets"): pad the groups with zero queries, whose outputs are + // dropped. Repro: + // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, + // platform/furiosa). Remove once any group size compiles. + const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); + const extra = groups - cfg.g; + cfg.g = groups; + const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); const inputs: OneShot.Inputs = .{ - .q = q.convert(.bf16), + .q = queries, .pages = table, .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)), + .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), }; - const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; + const outputs: OneShot.Outputs = .{ .out = queries.shape() }; const out = if (cfg.a == cfg.chunk) OneShot.call(inputs, outputs, .{ .cfg = cfg }).out else Flash.call(inputs, outputs, .{ .cfg = cfg }).out; - return out.convert(q.dtype()); + return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); } }; -/// Keys a decode kernel attends to in one pass: TCC 2026.5 fails on the -/// decode metadata for 131072 keys in pages of 16. +fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { + // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in + // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. + // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. + if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { + if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; + } + if (straight and cfg.window > 0) { + const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); + const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; + if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) + return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; + } + if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) + return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; + if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; +} + +const window_chunk_keys = 1024; +const window_flash_chunks = 16; +const batch_window_flash_chunks = 4; + +/// Widest block table the straight-line decode kernel takes for one token. +/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot +/// propagate gather bounds outside a loop over a token axis. Repro: +/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. +const straight_decode_keys = 32768; + +/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC +/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. +/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. const decode_keys = 65536; +const chunked_decode_tokens = 16; +const straight_batch_decode_keys = 8192; + +/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero +/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more +/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. +const unmasked_decode_keys = 1 << 17; + +const masked_decode_keys = 4096; + const DecodeCfg = struct { t: i64, a: i64, + group: i64, hkv: i64, g: i64, d: i64, @@ -132,14 +223,12 @@ const DecodeCfg = struct { pg: i64, scale: f32, is_causal: bool, + window: i64, + use_sink: bool, }; -/// Decode attention in Furiosa's form: the kernel gathers whole pages, then -/// runs one contraction, a masked softmax and a second contraction. It reads -/// past each sequence's end, selecting zero for those unused cache entries. -/// The token metadata depends only on the attention parameters, which every -/// layer shares: one kernel computes it once per forward pass. -fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, cfg: DecodeCfg) Tensor { +/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { const columns = @divExact(cfg.a, cfg.pg); const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ @@ -147,7 +236,7 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), - }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal } }); + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); return Decode.call(.{ .q = q.convert(.bf16), .pages = meta.pages, @@ -156,19 +245,15 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .keyed = meta.keyed, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), + .sink = sink, }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; } -const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool }; +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; -/// For each token: the cache page of each column, how many columns hold keys -/// it sees, the keys it sees (below its limit, in pages inside the cache) and -/// whether it sees any. Columns past the sequence's -/// pages read page `p`, so that the masked padding spreads over many pages -/// instead of hammering one. Rows are selected by one-hot products and -/// comparisons follow `attentionMask`'s clamp arithmetic: TCC 2026.5 has no -/// tactics for small integer gathers and fails on the plugin's lowering of the -/// same comparisons for several tokens. +/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC +/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, +/// repro_row_broadcast_past_4096.py. Remove once both compile. const DecodeMeta = tcl.Kernel(MetaCfg, .{ .name = "paged_decode_metadata", .inputs = &.{ "seq_lens", "query_offsets", "table" }, @@ -188,14 +273,12 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ }); const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - // Padding tokens follow every sequence's end and belong to none. const seq = try countBefore(b, ends, tokens, S); const own = try selector(b, seq, S, cfg.s); var op = b.tensorOperation(.{}); const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); var visible = try selectMetadata(b, t.seq_lens, own, S); if (cfg.is_causal) { - // Causal tokens see their sequence up to their own position. const end = try selectMetadata(b, ends, own, S); op = b.tensorOperation(.{}); visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); @@ -213,8 +296,6 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); op = b.tensorOperation(.{}); const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); - // Tokens without keys have no defined attention: the attention - // kernel zeroes them. op = b.tensorOperation(.{}); const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); op = b.tensorOperation(.{}); @@ -222,21 +303,19 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ op = b.tensorOperation(.{}); const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); op = b.tensorOperation(.{ .tactic = .elementwise }); - const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } b.ret(&.{ read, bounds, mask, keyed }); } }.run, }); -/// Gathers the key and value pages a token sees, contracts with the query, -/// masks keys past the sequence, normalizes in one softmax times `keyed` (zero -/// for tokens without keys, whose softmax is uniform) and contracts with the -/// values. A token's pages are gathered up to its bound, and the operations -/// reading them skip the rest: the work follows the sequence's length, not the -/// table's. Fusible: it joins the layer around it, one launch per layer. const Decode = tcl.Kernel(DecodeCfg, .{ .name = "paged_decode_attention", - .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache" }, + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, .outputs = &.{"out"}, .run = struct { fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { @@ -258,26 +337,360 @@ const Decode = tcl.Kernel(DecodeCfg, .{ .keyed = .{ .dtype = .f32, .axes = &.{T} }, .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, }); - const mask = try b.reshape(t.mask, &.{ T, AP, PG }, .{}); - const k = try maskCacheValues(b, try b.gather(t.k_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const v = try maskCacheValues(b, try b.gather(t.v_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const keys: []const tcl.Axis = &.{ AP, PG }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(t.q, k, &.{ Hkv, G, T, AP, PG }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); - op = b.tensorOperation(.{}); - const x = op.fetch(masked, .{}); - const e = x.subf(x.reduce(keys, .maxf)).exp(); - const p = try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(t.keyed, .{})), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 })}); + const sink = if (cfg.use_sink) t.sink else null; + const groups = @divExact(cfg.t, cfg.group); + if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); + const NG = b.axis("NG", groups); + const TG = b.axis("TG", @divExact(cfg.t, groups)); + var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); + { + const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); + const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); + const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); + const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); + const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); + const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); + } + b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); } }.run, }); +const unmasked_group_tokens = 4; + +/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. +fn decodeGroups(tokens: i64, keys: i64) i64 { + var groups: i64 = 1; + while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { + groups += 1; + while (@rem(tokens, groups) != 0) groups += 1; + } + return groups; +} + +/// `keyed` scales the softmax: zero for tokens without keys. +fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const AP = pages.axes[1]; + const P = k_cache.axes[0]; + const PG = k_cache.axes[1]; + const keys: []const tcl.Axis = &.{ AP, PG }; + const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); + var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { + k = try maskCacheValues(b, k, key_mask); + v = try maskCacheValues(b, v, key_mask); + } + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const p = if (sink) |logits| blk: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); + op = b.tensorOperation(.{}); + const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); + op = b.tensorOperation(.{}); + break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + } else blk: { + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); +} + +const head_keys = 4096; +const tail_chunk_keys = 8192; + +const HybridCfg = struct { + t: i64, + s: i64, + /// Head pages, tail chunks and pages per tail chunk. + hp: i64, + nc: i64, + cp: i64, + pg: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + scale: f32, + is_causal: bool, +}; + +fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { + const pg = k_cache.dim(.k_chunk); + const t = q.dim(.b); + const s = parameters.seq_lens.dim(.b); + const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; + const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; + const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; + const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); + const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), + .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), + .hbounds = zml.Shape.init(.{ .t = t }, .i32), + .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), + .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), + .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), + .limits = zml.Shape.init(.{ .t = t }, .i32), + .keyed = zml.Shape.init(.{ .t = t }, .f32), + }, .{ .cfg = cfg }); + return HybridDecode.call(.{ + .q = q.convert(.bf16), + .hpages = meta.hpages, + .hmask = meta.hmask, + .hbounds = meta.hbounds, + .tpages = meta.tpages, + .tmask = meta.tmask, + .tbounds = meta.tbounds, + .limits = meta.limits, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the +/// longest token; `limits` is the visible key count. +const HybridMeta = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); + const HP = b.axis("HP", cfg.hp); + const RP = b.axis("RP", cfg.nc * cfg.cp); + const NC = b.axis("NC", cfg.nc); + const CP = b.axis("CP", cfg.cp); + const HA = b.axis("HA", cfg.hp * cfg.pg); + const CK = b.axis("CK", cfg.cp * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); + + // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read + // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. + // Remove once it is exact on RNGD. + const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + op = b.tensorOperation(.{}); + const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); + // Trips never exceed nc: `limit` is at most the table's keys. + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); + { + const reads, const masks, const counts, const seen = loop.carried; + const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); + const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); + try loop.yield(.{ + try b.indexWrite(reads, loop.iv, chunk.pages), + try b.indexWrite(masks, loop.iv, chunk.mask), + try b.indexWrite(counts, loop.iv, chunk.bounds), + any, + }); + } + const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); + b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + } + }.run, +}); + +/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. +fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { + const T = own.axes[0]; + const S = own.axes[1]; + const PG = b.axis("PG", cfg.pg); + const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); + var op = b.tensorOperation(.{}); + const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); + op = b.tensorOperation(.{}); + const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + const to: []const tcl.Axis = &.{ T, cols, PG }; + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); + return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; +} + +const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; + +const HybridDecode = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_decode_attention", + .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + b.fuse(); + const a = hybridAxes(b, cfg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, + .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, + .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, + .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, + .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, + .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, + .limits = .{ .dtype = .i32, .axes = &.{a.T} }, + .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + }); + const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); + b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); + } + }.run, +}); + +const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; + +fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { + return .{ + .T = b.axis("T", cfg.t), + .HP = b.axis("HP", cfg.hp), + .HA = b.axis("HA", cfg.hp * cfg.pg), + .NC = b.axis("NC", cfg.nc), + .CP = b.axis("CP", cfg.cp), + .CK = b.axis("CK", cfg.cp * cfg.pg), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; +} + +fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { + const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); + { + const max, const sum, const out = loop.carried; + const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); + const bounds = try b.indexRead(tbounds, loop.iv); + const pages = try b.indexRead(tpages, loop.iv); + const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); + try loop.yield(.{ merged.max, merged.sum, merged.out }); + } + return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; +} + +fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { + const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); + const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const keys: []const tcl.Axis = &.{ a.HP, a.PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); + const alpha = try expDifference(b, tail.max, max); + op = b.tensorOperation(.{}); + const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); + op = b.tensorOperation(.{}); + const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); +} + +fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const keys = mask.axes[1..]; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); + const exps = try maskedExp(b, masked, max, mask); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); + const alpha = try expDifference(b, x.max, max); + const beta = try expDifference(b, y.max, max); + op = b.tensorOperation(.{}); + const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); + op = b.tensorOperation(.{}); + const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + const Cfg = struct { tokens: i64, sequences: i64, @@ -299,11 +712,9 @@ const Cfg = struct { window: i32 = -1, }; -/// Attention tuning inherited from the original TCL backend, informed by -/// `furiosa.kernels.llama.config` and `common.attention`. Keep the compiler's -/// default lowering mode and omit the adaptive-einsum and half-VRF settings -/// removed in TCC 2026.5. +/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. fn configure(b: *tcl.Builder, cfg: Cfg) void { + const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; if (cfg.decode) b.compilerConfig(.{ .enable_einsum_fusion = true, .padding_policy = .{ .Small = 1.1 }, @@ -326,11 +737,10 @@ fn configure(b: *tcl.Builder, cfg: Cfg) void { .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, .tensor_unit_bridge_threshold_in_page = 12, .allow_external_operators = false, - .enable_tactic_pruning = false, .scheduler_beam_search = true, .allow_reduce_by_ve_cluster_chip_reduce = false, .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, .dma_preference = 1.2, }); } @@ -390,11 +800,24 @@ fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { const Prepared = struct { q: builder.Tensor, + /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). pages: builder.Tensor, + pick: ?builder.Tensor, qpos: builder.Tensor, kvlen: builder.Tensor, row: builder.Tensor, active: builder.Tensor, + /// Per page row: the first query position and the keys. + first: builder.Tensor, + keys: builder.Tensor, + + /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash + /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such + /// products compile. + fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { + const pick = self.pick orelse return pages; + return selectMetadata(b, pages, pick, a.S); + } }; fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { @@ -414,15 +837,16 @@ fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64 return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); } -// Keep metadata exact in I32. Scalar gathers and four-byte-offset slices do -// not have native tactics for these shapes in TCC 2026.5. +// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for +// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, +// repro_i32_slice_odd_offset.py. Remove once both compile. fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); } -/// Keep adjacent I32 reads inside the native unit. Exposing offsets[1:] as a -/// separate argument would require alignment staging at every invocation. +/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would +/// need alignment staging at every invocation. fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); @@ -432,8 +856,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); const seq_of_token = try countBefore(b, ends, tokens, a.S); - // All sequence ends precede a padding token. Reuse this vector instead of - // bringing a symbolic scalar reduction back into a vector operation. + // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. op = b.tensorOperation(.{}); const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); op = b.tensorOperation(.{}); @@ -453,16 +876,19 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis return .{ .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .pick = null, .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), .kvlen = try b.reshape(lengths, &.{a.B}, .{}), .row = token_seq, .active = active, + .first = try b.reshape(qpos, &.{a.B}, .{}), + .keys = try b.reshape(lengths, &.{a.B}, .{}), }; } - const shift = std.math.log2_int(u64, prefill_block); + const shift = std.math.log2_int(u64, @intCast(cfg.i)); op = b.tensorOperation(.{}); - const blocks = try op.commit(op.fetch(qlen, .{}).addi(prefill_block - 1).binary(.shr_arith, shift), .{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); op = b.tensorOperation(.{}); const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); op = b.tensorOperation(.{}); @@ -482,7 +908,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); op = b.tensorOperation(.{}); - const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(prefill_block), .{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); op = b.tensorOperation(.{}); @@ -500,14 +926,15 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); op = b.tensorOperation(.{}); const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); - const table = try selectMetadata(b, t.pages, pick, a.S); const token_pick = try selector(b, token_seq, a.S, cfg.sequences); const token_begin = try selectMetadata(b, starts, token_pick, a.S); const token_block = try selectMetadata(b, block_start, token_pick, a.S); op = b.tensorOperation(.{}); - const row = try op.commit(op.fetch(token_block, .{}).muli(prefill_block).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); - return .{ .q = q, .pages = table, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active }; + const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + op = b.tensorOperation(.{}); + const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); + return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; } fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { @@ -533,8 +960,7 @@ fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.Finish return .{ .safe = safe, .valid = valid }; } -/// Cache validity is per sequence, not per query: causal/window masking stays -/// on scores, without replicating K/V for every row of a prefill block. +/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { var op = b.tensorOperation(.{}); const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); @@ -542,17 +968,15 @@ fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); } -/// A true select excludes NaN/Inf in unwritten slots; multiplying by zero -/// would still propagate them into the contraction. Materialize only BF16. +/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = .elementwise }); const data = op.fetch(values, .{}); return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); } -/// Build the sequence, query, page and window mask for the current keys. fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { - const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[0] }; + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; // Elementwise operations require their first read to cover the output // axes, and each VE instruction must consume the current accumulator. @@ -577,8 +1001,6 @@ fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder return mask; } -/// Keep the contraction accumulator in f32 through masking and softmax. -/// A finite sentinel also makes entirely masked chunks safe to rescale. fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; var op = b.tensorOperation(.{}); @@ -591,8 +1013,7 @@ fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builde fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{}); const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); - // Subtracting the sentinel from itself gives exp(0), so masking only - // the logits is insufficient for empty rows and fully masked chunks. + // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); } @@ -622,7 +1043,7 @@ const OneShot = tcl.Kernel(Cfg, .{ const prepared = try prepareQueries(b, cfg, a, t); const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const page_indices = try validIndices(b, prepared.pages, cfg.pages); + const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); @@ -630,7 +1051,6 @@ const OneShot = tcl.Kernel(Cfg, .{ slot_op = b.tensorOperation(.{}); const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); - // Match the original slot sanitization before reading invalid pages. slot_op = b.tensorOperation(.{}); const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); const valid_length = if (cfg.decode) prepared.kvlen else null; @@ -642,8 +1062,6 @@ const OneShot = tcl.Kernel(Cfg, .{ const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); var op = b.tensorOperation(.{}); const p = if (cfg.decode) compact: { - // Furiosa's common.softmax keeps normalization inside one VE - // operation. FP32 intermediates only narrow at the DPE input. const exp_sink = if (cfg.use_sink) sink: { const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); break :sink try expDifference(b, t.sink, maximum); @@ -656,13 +1074,11 @@ const OneShot = tcl.Kernel(Cfg, .{ var sum = e.reduce(&.{A}, .addf); if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); - // Empty rows have uniform probabilities from finite sentinels. - // Clear masked probabilities before contracting with values. + // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. op = b.tensorOperation(.{}); break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); } else tiled: { - // Separate reductions schedule better for multi-query blocks - // with the current compiler; keep their intermediate state FP32. + // Separate reductions schedule better for multi-query blocks with the current compiler. var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); const maximum = try op.commit(m, .{}); @@ -695,28 +1111,66 @@ const Flash = tcl.Kernel(Cfg, .{ const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); const t = try declareInputs(b, a); const prepared = try prepareQueries(b, cfg, a, t); - const by_batch = try b.reshape(prepared.pages, &.{ a.B, C, Cp }, .{}); + const rows = prepared.pages.axes[0]; + const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); const page_op = b.tensorOperation(.{}); - const pages = try page_op.commit(page_op.fetch(by_batch, .{}), .{ .axes = &.{ C, a.B, Cp } }); + const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); const stats = &.{ a.Hkv, a.G, a.B, a.I }; const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; var op = b.tensorOperation(.{}); - const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); + // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides + // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. + const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { + op = b.tensorOperation(.{}); + const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); + // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per + // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index + // compiles as data. + const trip = try b.arange(C.size, .i32, &.{C}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); + break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; + } else null; + const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); { const sum_acc, const qkv_acc, const max_acc = loop.carried; - const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + var row_pages: builder.Tensor = undefined; + var position: builder.Tensor = undefined; + var first_keys: builder.Tensor = undefined; + if (window) |w| { + const chunk_index = try b.indexRead(w.chunks, loop.iv); + op = b.tensorOperation(.{}); + const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); + // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower + // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove + // once batched gathers of several rows compile. + row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); + op = b.tensorOperation(.{}); + const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); + const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } else { + row_pages = try b.indexRead(pages, loop.iv); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } + const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); const idx = indices.safe; op = b.tensorOperation(.{}); const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - op = b.tensorOperation(.{}); - const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); op = b.tensorOperation(.{}); @@ -724,9 +1178,8 @@ const Flash = tcl.Kernel(Cfg, .{ op = b.tensorOperation(.{}); const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); - const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); - const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); op = b.tensorOperation(.{}); const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); @@ -751,7 +1204,6 @@ const Flash = tcl.Kernel(Cfg, .{ var sum = loop.results[0]; var numerator = loop.results[1]; if (cfg.use_sink) { - // Merge the zero-valued sink once, after all key chunks. op = b.tensorOperation(.{}); const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); const alpha = try expDifference(b, loop.results[2], maximum); @@ -922,6 +1374,7 @@ const TestDims = struct { page: usize = 16, uniform: bool = false, uniform_scores: bool = false, + spikes: []const usize = &.{}, dtype: zml.DataType = .bf16, }; @@ -1030,7 +1483,7 @@ test "tcl paged attention cache masking emits a BF16 select without query replic try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.ve_opcode<\"mulf\">") == null); try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); - const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true }); + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); defer std.testing.allocator.free(decode); try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); @@ -1143,6 +1596,32 @@ test "tcl paged attention chunked decode on furiosa" { try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); } +test "tcl paged attention hybrid decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(2049, 2101, &.{ + .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, + }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); +} + +test "tcl paged attention flash decode of many tokens on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); + try runCases(2049, 2600, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2700, &.{ + .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, + }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); +} + test "tcl paged attention uniform probabilities on furiosa" { if (zml.testing.env().target != .furiosa) return error.SkipZigTest; try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); @@ -1172,11 +1651,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { - return paged_attention.pagedAttention(.{ .tcl = p }, q, k, v, .{ .split = .{ .k = k, .v = v } }, opts); + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { + return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: paged_attention.AttentionOptions = .{ + const opts: AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1209,9 +1688,16 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (dims.spikes.len > 0) { + for (case.seqs[0..batch], 0..) |seq, row| { + for (0..@intCast(seq[1])) |key| { + const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); + const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; + @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); + } + } + } if (case.poison != .none) { - // Mark the union of live physical slots before poisoning: reordered - // or shared pages must retain every token a live sequence can read. const live = try allocator.alloc(bool, num_pages * page); defer allocator.free(live); @memset(live, false); @@ -1243,11 +1729,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ + var opts_d: zml.Bufferized(AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 1ccbeba8..2a12e012 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,10 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -274,92 +274,57 @@ pub const paged = struct { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); } - const output = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q.dim(.b)); + const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; + const sliding_window: usize = opts.mask.slidingWindow() orelse 0; - const cu_count = getCuCount(); - const num_heads: usize = @intCast(self.q.dim(.hkv) * self.q.dim(.hg)); - const num_kv_heads: usize = @intCast(self.k_cache.dim(.hkv)); - const num_queries_per_kv: usize = num_heads / num_kv_heads; - // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the - // single decode query token doesn't carry masked-out fp32 acc lanes. - // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) - std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) - else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(self.q.dim(.b)); - const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); - const total_q_blocks: usize = num_tokens / block_q + num_seqs; - const target_num_prgms: usize = cu_count * 4; - const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !parameters.options_.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q.dim(.hd)), + .batch_size = @intCast(parameters.block_table.dim(.b)), + .block_size = @intCast(k_cache.dim(.k_chunk)), + .num_blocks = @intCast(k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = parameters.options_.max_seqlen_q, + .scale = opts.scale, + }; - const sliding_window: usize = self.opts.mask.slidingWindow() orelse 0; - - const paged_attention_opts: PagedAttentionOptions = .{ - .cu_count = getCuCount(), - .all_decode = !self.options.is_prefill, - .num_tokens = num_tokens, - .num_heads = num_heads, - .num_kv_heads = num_kv_heads, - .head_dim = @intCast(self.q.dim(.hd)), - .batch_size = @intCast(parameters_.block_table.dim(.b)), - .block_size = @intCast(self.k_cache.dim(.k_chunk)), - .num_blocks = @intCast(self.k_cache.dim(.page)), - .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), - .sliding_window = sliding_window, - .block_m = block_m, - .block_q = block_q, - .total_q_blocks = total_q_blocks, - .target_num_prgms = target_num_prgms, - .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = self.options.max_seqlen_q, - .scale = self.opts.scale, - }; - - const use_2d_kernel = use2dKernel( - paged_attention_opts.all_decode, - paged_attention_opts.batch_size, - paged_attention_opts.num_kv_heads, - ); - const output = if (use_2d_kernel) - pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else - pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); - - return output; - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else + pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); return output; } @@ -372,51 +337,18 @@ pub const paged = struct { stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; - - const config = selectSparseConfig( - @intCast(self.opts.mask.indices.dim(.topk)), - @intCast(self.block_table.dim(.b)), - @intCast(self.q.dim(.hkv)), - getCuCount(), - !self.options.is_prefill, - ); - - return if (config.num_segments == 1) - pagedSparseAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config) - else - pagedSparseAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config); - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const config = selectSparseConfig( + @intCast(opts.mask.indices.dim(.topk)), + @intCast(parameters.block_table.dim(.b)), + @intCast(q.dim(.hkv)), + getCuCount(), + !parameters.options_.is_prefill, ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) + else + pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); } pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index 39018f1b..7b2b9428 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -1020,6 +1020,25 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + + // The launch covers the padded query capacity, but the attention pass only + // writes scratch for active tokens. Zero the tail before looking up a + // sequence or dividing by its segment size (zero for inactive sequences). + const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); + const inactive = query_token_idx.ge(active_tokens); + k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ + .mask = dim_mask.bitAnd(inactive), + }); + k.returnIf(inactive); + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); @@ -1029,12 +1048,6 @@ fn reduceSegments( const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); @@ -1100,9 +1113,6 @@ fn reduceSegments( ); } - const output_offset = query_token_idx.to(.i64).mul(output_stride_0) - .add(query_head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); k.storeOpts( output_ptr.addPtr(output_offset), acc.to(config.o_dtype), diff --git a/zml/mem.zig b/zml/mem.zig index 596ebe85..a64ae968 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -1,9 +1,320 @@ const std = @import("std"); +const Allocator = std.mem.Allocator; +const Alignment = std.mem.Alignment; +const assert = std.debug.assert; + +const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; +const Device = @import("platform.zig").Device; +const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); +const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; +const log = std.log.scoped(.@"zml/mem"); + +pub const DmaAllocator = union(enum) { + passthrough: std.mem.Allocator, + uib: UninitializedBufferAllocator, + dmam: DmaMapAllocator, + + pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { + return switch (device.platform.target) { + .cuda => .{ .dmam = .init(parent, device.platform) }, + .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, + .cpu, .neuron, .metal => .{ .passthrough = parent }, + }; + } + + pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { + return switch (self.*) { + .passthrough => |a| a, + inline else => |*a| a.allocator(), + }; + } +}; + +pub const UninitializedBufferAllocator = struct { + memory: *const Memory, + + const Header = struct { + buffer: *pjrt.Buffer, + }; + + pub fn init(memory: *const Memory) UninitializedBufferAllocator { + return .{ + .memory = memory, + }; + } + + pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; + const pjrt_client = self.memory.platform.pjrt_client; + + const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); + + const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ + .dims = &.{@intCast(total_len)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = self.memory.pjrt_memory }, + }) catch return null; + + const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); + const data_with_header: []u8 = opaque_ptr[0..total_len]; + + const header = std.mem.bytesAsValue(Header, opaque_ptr); + header.* = .{ + .buffer = pjrt_buffer, + }; + const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); + return @ptrCast(data_with_header[offset..]); + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { + _ = ret_addr; + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; + const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); + header.buffer.deinit(pjrt_api); + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } +}; + +pub const DmaMapAllocator = struct { + parent: std.mem.Allocator, + platform: *const Platform, + + pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { + return .{ + .parent = parent, + .platform = platform, + }; + } + + pub fn allocator(self: *const DmaMapAllocator) Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + const allocation = self.parent.rawAlloc(len, alignment, ret_addr); + if (allocation) |loc| { + const data = loc[0..len]; + self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { + self.parent.rawFree(data, alignment, ret_addr); + return null; + }; + } + return allocation; + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; + self.parent.rawFree(buf, alignment, ret_addr); + } +}; + +pub const FixedBufferPool = struct { + buffer: []u8, + block_size: usize, + q_buf: []const u16, + q: std.Io.Queue(u16), + + pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { + const block_size = @divExact(buffer_.len, blocks_); + const q_buf = try allocator.alloc(u16, blocks_); + for (q_buf, 0..) |*idx, i| { + idx.* = @intCast(i); + } + var q: std.Io.Queue(u16) = .init(q_buf); + q.type_erased.len = q.type_erased.buffer.len; // make the queue full + return .{ + .buffer = buffer_, + .block_size = block_size, + .q = q, + .q_buf = q_buf, + }; + } + + pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { + allocator.free(self.q_buf); + } + + pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { + const idx = try self.q.getOneUncancelable(io); + return self.buffer[idx * self.block_size ..][0..self.block_size]; + } + + fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { + return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and + @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); + } + + pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { + // is the pointer in range ? + std.debug.assert(inRange(buf, self.buffer)); + const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); + self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; + } +}; + +pub const DynamicBufferPool = struct { + const Node = struct { next: ?*Node }; + const alignment: std.mem.Alignment = .of(Node); + + block_size: usize, + max_blocks: usize, + + arena_state: std.heap.ArenaAllocator.State = .{}, + free_stack: std.atomic.Value(?*Node) = .init(null), + in_flight: std.atomic.Value(usize) = .init(0), + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { + return .{ + .block_size = block_size, + .max_blocks = max_blocks, + }; + } + + pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { + self.arena_state.promote(allocator).deinit(); + } + + pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.max_blocks) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } + + if (self.pop()) |node| { + return self.nodeToSlice(node); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + var arena = self.arena_state.promote(allocator); + defer self.arena_state = arena.state; + + errdefer _ = self.in_flight.fetchSub(1, .release); + const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); + return @alignCast(buffer); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.max_blocks) { + try self.condition.wait(io, &self.mutex); + } + } + } + + pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { + const node: *Node = @ptrCast(@alignCast(buf.ptr)); + var head = self.free_stack.load(.acquire); + while (true) { + node.next = head; + if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { + head = actual_head; + } else { + break; + } + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.condition.signal(io); + } + + fn pop(self: *DynamicBufferPool) ?*Node { + var head = self.free_stack.load(.acquire); + while (true) { + if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { + head = actual; + continue; + } + + return head; + } + } + + fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { + const ptr: [*]u8 = @ptrCast(node); + return ptr[0..self.block_size]; + } +}; + /// Return a clone of a type with Tensors replaced by Buffer. /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. @@ -12,21 +323,6 @@ pub fn Bufferized(comptime T: type) type { return meta.MapRestrict(Tensor, Buffer).map(T); } -/// The field of `Bufferized(Model)` that holds the struct `Model`'s field -/// `name`, or null when `Bufferized` strips it. `Bufferized` keeps the -/// fields holding tensors, in order, and renumbers a tuple's. -fn bufferizedField(comptime Model: type, comptime name: []const u8) ?[]const u8 { - @setEvalBranchQuota(10_000); - const info = @typeInfo(Model).@"struct"; - var index: usize = 0; - for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { - if (attrs.@"comptime" or !meta.Contains(Field, Tensor)) continue; - if (std.mem.eql(u8, field_name, name)) return @typeInfo(Bufferized(Model)).@"struct".field_names[index]; - index += 1; - } - return null; -} - fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { @setEvalBranchQuota(10_000); const Model = @TypeOf(model); @@ -45,16 +341,8 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - var initialized_fields: usize = 0; - errdefer inline for (struct_type_info.field_names, 0..) |field_name, index| { - if (index < initialized_fields) - deinitBufferized(allocator, &@field(bufferized_, field_name)); - }; - inline for (@typeInfo(Model).@"struct".field_names) |field_name| { - if (comptime bufferizedField(Model, field_name)) |buffer_field| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, buffer_field)); - initialized_fields += 1; - } + inline for (struct_type_info.field_names) |field_name| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); } }, .@"union" => { @@ -76,333 +364,23 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - const allocated = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); - var initialized: usize = 0; - errdefer { - for (allocated[0..initialized]) |*element| deinitBufferized(allocator, element); - allocator.free(allocated); - } - for (model, allocated) |src, *dst| { + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); - initialized += 1; } - bufferized_.* = allocated; }, else => unreachable, } }, - .array => |info| { - var initialized: usize = 0; - errdefer for (bufferized_.*[0..initialized]) |*element| deinitBufferized(allocator, element); - inline for (0..info.len) |index| { - try bufferizeInner(allocator, model[index], &bufferized_.*[index]); - initialized = index + 1; - } - }, .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, else => unreachable, } } -/// Convert the model `model` points to into its bufferized form by replacing -/// Tensor fields with Buffer and allocating any required slices using the -/// provided allocator. -pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model.*)) { - var bufferized: Bufferized(@TypeOf(model.*)) = undefined; +/// Convert a model to its bufferized form by replacing Tensor fields with Buffer +/// and allocating any required slices using the provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { + var bufferized: Bufferized(ModelType) = undefined; try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } - -/// A tensor of a model, the buffer at its place in the model's buffers, and -/// its field path from the root, as `.layers[1].ffn.weight`. -pub const Pair = struct { tensor: *const Tensor, buffer: *Buffer, path: []const u8 }; - -/// Pairs each tensor of the model `model` points to with the buffer at its -/// place in `bufferized`, in the order `meta.visit` finds the tensors. The -/// pairs and their paths live in `arena`. The two trees must have the -/// structure `bufferize` gives: the same union arms, the same optionals set, -/// slices of the same lengths. `report`, an empty fixed writer, builds the -/// paths and cuts them short when full. At the first place the trees differ -/// the walk stops and returns `error.Mismatch`, and `report` ends with the -/// path there and what differs. -pub fn pairBufferized( - arena: std.mem.Allocator, - model: anytype, - bufferized: *Bufferized(@TypeOf(model.*)), - report: *std.Io.Writer, -) error{ Mismatch, OutOfMemory }![]Pair { - var pairs: std.ArrayList(Pair) = try .initCapacity(arena, meta.count(Tensor, model)); - try pairBufferizedInner(arena, model, bufferized, report, &pairs); - return pairs.items; -} - -/// `report.end` is restored after each part that pairs, so at a tensor or a -/// mismatch the report holds the path down to it. -fn pairBufferizedInner( - arena: std.mem.Allocator, - model: anytype, - bufferized: anytype, - report: *std.Io.Writer, - pairs: *std.ArrayList(Pair), -) error{ Mismatch, OutOfMemory }!void { - const Model = @typeInfo(@TypeOf(model)).pointer.child; - if (comptime !meta.Contains(Model, Tensor)) return; - if (Model == Tensor) return pairs.appendAssumeCapacity(.{ - .tensor = model, - // `bufferize` allocates every slice mutable, a `[]const` one included. - .buffer = @constCast(bufferized), - .path = try arena.dupe(u8, report.buffered()), - }); - - const depth = report.end; - switch (@typeInfo(Model)) { - .@"struct" => |info| inline for (info.field_names) |name| { - if (comptime bufferizedField(Model, name)) |buffer_field| { - report.print(".{s}", .{name}) catch {}; - try pairBufferizedInner(arena, &@field(model, name), &@field(bufferized, buffer_field), report, pairs); - report.end = depth; - } - }, - .@"union" => { - const tag = std.meta.activeTag(model.*); - if (tag != std.meta.activeTag(bufferized.*)) { - report.print(": union arm .{t} in the model, .{t} in the buffers", .{ tag, std.meta.activeTag(bufferized.*) }) catch {}; - return error.Mismatch; - } - switch (model.*) { - inline else => |*payload, arm| { - report.print(".{s}", .{@tagName(arm)}) catch {}; - try pairBufferizedInner(arena, payload, &@field(bufferized, @tagName(arm)), report, pairs); - report.end = depth; - }, - } - }, - .optional => { - if ((model.* == null) != (bufferized.* == null)) { - report.print(": {s} in the model, {s} in the buffers", .{ if (model.* == null) "null" else "set", if (bufferized.* == null) "null" else "set" }) catch {}; - return error.Mismatch; - } - if (model.*) |*payload| try pairBufferizedInner(arena, payload, &bufferized.*.?, report, pairs); - }, - .pointer => |info| { - if (info.size != .slice) @compileError("mem.pairBufferized: " ++ @typeName(Model) ++ " is not a slice"); - if (model.len != bufferized.len) { - report.print(": a slice of {d} in the model, of {d} in the buffers", .{ model.len, bufferized.len }) catch {}; - return error.Mismatch; - } - for (model.*, bufferized.*, 0..) |*element, *buffer, i| { - report.print("[{d}]", .{i}) catch {}; - try pairBufferizedInner(arena, element, buffer, report, pairs); - report.end = depth; - } - }, - .array => for (model, bufferized, 0..) |*element, *buffer, i| { - report.print("[{d}]", .{i}) catch {}; - try pairBufferizedInner(arena, element, buffer, report, pairs); - report.end = depth; - }, - else => @compileError("mem.pairBufferized: cannot walk " ++ @typeName(Model)), - } -} - -/// Deinitializes every accelerator buffer of the tree `value` points to and -/// frees the recursive slice storage allocated by `bufferize`. -pub fn deinitBufferized(allocator: std.mem.Allocator, value: anytype) void { - const Ptr = @TypeOf(value); - const T = @typeInfo(Ptr).pointer.child; - if (T == Tensor) @compileError("mem.deinitBufferized takes the buffers bufferize returns, not the model"); - if (T == Buffer) { - const buffer: *Buffer = @constCast(value); - buffer.deinit(); - return; - } - switch (@typeInfo(T)) { - .@"struct" => |info| inline for (info.field_names) |field_name| { - deinitBufferized(allocator, &@field(value, field_name)); - }, - .@"union" => switch (value.*) { - inline else => |*payload| deinitBufferized(allocator, payload), - }, - .optional => if (value.*) |*payload| { - deinitBufferized(allocator, payload); - }, - .pointer => |info| switch (info.size) { - .slice => { - for (value.*) |*element| deinitBufferized(allocator, element); - allocator.free(value.*); - }, - else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), - }, - .array => for (&value.*) |*element| deinitBufferized(allocator, element), - // What `Bufferized` makes of a union arm or a whole tree without a tensor. - .void => {}, - else => @compileError("mem.deinitBufferized takes the buffers bufferize returns, not " ++ @typeName(T)), - } -} - -test "bufferize maps a tuple's fields as Bufferized renumbers them" { - const Model = struct { u32, Tensor, []Tensor, [2]Tensor }; - const a: Tensor = .init(.{2}, .f32); - const b: Tensor = .init(.{3}, .f16); - var tensors = [_]Tensor{ a, b }; - const model: Model = .{ 3, a, &tensors, .{ b, a } }; - var bufferized = try bufferize(std.testing.allocator, &model); - defer deinitBufferized(std.testing.allocator, &bufferized); - try std.testing.expect(bufferized[0].shape().eql(a.shape())); - try std.testing.expectEqual(2, bufferized[1].len); - try std.testing.expect(bufferized[1][1].shape().eql(b.shape())); - try std.testing.expect(bufferized[2][0].shape().eql(b.shape())); - try std.testing.expect(bufferized[2][1].shape().eql(a.shape())); -} - -test "bufferize rolls back earlier slice fields on allocation failure" { - const Model = struct { - first: []const Tensor, - second: []const Tensor, - }; - var tensors: [1]Tensor = undefined; - const model: Model = .{ .first = &tensors, .second = &tensors }; - var failing: std.testing.FailingAllocator = .init(std.testing.allocator, .{ .fail_index = 1 }); - - if (bufferize(failing.allocator(), &model)) |result| { - var unexpected = result; - deinitBufferized(failing.allocator(), &unexpected); - return error.ExpectedOutOfMemory; - } else |err| { - try std.testing.expect(err == error.OutOfMemory); - } - try std.testing.expect(failing.has_induced_failure); - try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); -} - -test "deinitBufferized frees recursive const slices" { - const Layer = struct { weights: []const Tensor }; - const Model = struct { - layers: []const Layer, - fixed: [2]Tensor, - }; - var weights: [2]Tensor = undefined; - const layers = [_]Layer{ - .{ .weights = weights[0..1] }, - .{ .weights = weights[1..2] }, - }; - const model: Model = .{ .layers = &layers, .fixed = undefined }; - var bufferized = try bufferize(std.testing.allocator, &model); - deinitBufferized(std.testing.allocator, &bufferized); -} - -test "deinitBufferized frees slices under optionals and unions" { - const Layer = struct { weights: []const Tensor }; - const Ffn = union(enum) { dense: Layer, experts: []const Layer, none: void }; - const Model = struct { - present: ?Layer, - absent: ?Layer, - ffns: [3]Ffn, - }; - var weights: [3]Tensor = undefined; - const experts = [_]Layer{ .{ .weights = weights[0..1] }, .{ .weights = weights[1..3] } }; - const model: Model = .{ - .present = .{ .weights = &weights }, - .absent = null, - .ffns = .{ .{ .dense = .{ .weights = weights[2..3] } }, .{ .experts = &experts }, .none }, - }; - var bufferized = try bufferize(std.testing.allocator, &model); - deinitBufferized(std.testing.allocator, &bufferized); -} - -test "pairBufferized pairs tensors and buffers in visit order" { - const Layer = struct { weights: []Tensor, scale: ?Tensor }; - const Ffn = union(enum) { dense: Layer, experts: []Layer, none: void }; - const Model = struct { ffns: [3]Ffn, head: Tensor, layers: u32 }; - var weights: [4]Tensor = undefined; - var experts = [_]Layer{ .{ .weights = weights[1..2], .scale = null }, .{ .weights = weights[2..4], .scale = weights[3] } }; - var model: Model = .{ - .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = weights[0] } }, .{ .experts = &experts }, .none }, - .head = undefined, - .layers = 2, - }; - var bufferized = try bufferize(std.testing.allocator, &model); - defer deinitBufferized(std.testing.allocator, &bufferized); - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - var report_buffer: [64]u8 = undefined; - var report: std.Io.Writer = .fixed(&report_buffer); - const pairs = try pairBufferized(arena.allocator(), &model, &bufferized, &report); - try std.testing.expectEqual(0, report.end); - - const Visited = struct { - tensors: [8]*const Tensor = undefined, - buffers: [8]*Buffer = undefined, - tensor_count: usize = 0, - buffer_count: usize = 0, - }; - var visited: Visited = .{}; - meta.forEachVisit(&model, *const Tensor, struct { - fn call(_: usize, tensor: *const Tensor, out: *Visited) void { - out.tensors[out.tensor_count] = tensor; - out.tensor_count += 1; - } - }.call, .{&visited}); - meta.forEachVisit(&bufferized, *Buffer, struct { - fn call(_: usize, buffer: *Buffer, out: *Visited) void { - out.buffers[out.buffer_count] = buffer; - out.buffer_count += 1; - } - }.call, .{&visited}); - const paths = [_][]const u8{ - ".ffns[0].dense.weights[0]", - ".ffns[0].dense.scale", - ".ffns[1].experts[0].weights[0]", - ".ffns[1].experts[1].weights[0]", - ".ffns[1].experts[1].weights[1]", - ".ffns[1].experts[1].scale", - ".head", - }; - try std.testing.expectEqual(paths.len, pairs.len); - try std.testing.expectEqual(pairs.len, visited.tensor_count); - try std.testing.expectEqual(pairs.len, visited.buffer_count); - // Not `expectEqualSlices`: printing a `*Buffer` reaches code a CPU test build lacks. - for (pairs, paths, 0..) |pair, path, i| { - try std.testing.expect(pair.tensor == visited.tensors[i]); - try std.testing.expect(pair.buffer == visited.buffers[i]); - try std.testing.expectEqualStrings(path, pair.path); - } -} - -test "pairBufferized stops at the first place the trees differ and names it" { - const Layer = struct { weights: []Tensor, scale: ?Tensor }; - const Ffn = union(enum) { dense: Layer, none: void }; - const Model = struct { ffns: []Ffn, head: Tensor }; - var weights: [3]Tensor = undefined; - var built = [_]Ffn{ .{ .dense = .{ .weights = weights[0..2], .scale = null } }, .none }; - const built_model: Model = .{ .ffns = &built, .head = undefined }; - var bufferized = try bufferize(std.testing.allocator, &built_model); - defer deinitBufferized(std.testing.allocator, &bufferized); - - const Case = struct { ffns: [2]Ffn, report: []const u8 }; - const cases = [_]Case{ - .{ .ffns = .{ .none, .none }, .report = ".ffns[0]: union arm .none in the model, .dense in the buffers" }, - .{ - .ffns = .{ .{ .dense = .{ .weights = weights[0..2], .scale = weights[2] } }, .none }, - .report = ".ffns[0].dense.scale: set in the model, null in the buffers", - }, - .{ - .ffns = .{ .{ .dense = .{ .weights = weights[0..1], .scale = null } }, .none }, - .report = ".ffns[0].dense.weights: a slice of 1 in the model, of 2 in the buffers", - }, - }; - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - for (cases) |case| { - var ffns = case.ffns; - const model: Model = .{ .ffns = &ffns, .head = undefined }; - var report_buffer: [128]u8 = undefined; - var report: std.Io.Writer = .fixed(&report_buffer); - // Not `expectError`: printing a `*Buffer` reaches code a CPU test build lacks. - if (pairBufferized(arena.allocator(), &model, &bufferized, &report)) |_| { - return error.TestExpectedMismatch; - } else |err| try std.testing.expectEqual(error.Mismatch, err); - try std.testing.expectEqualStrings(case.report, report.buffered()); - } -} diff --git a/zml/meta.zig b/zml/meta.zig index 504b2973..84fb9d36 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -143,7 +143,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -180,6 +180,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } + if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { + to.* = from; + return; + } + if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { @@ -226,7 +231,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + else => { + const item = try allocator.create(ptr_info.child); + try mapAlloc(cb, allocator, ctx, from.*, item); + to.* = item; + }, }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -267,6 +276,7 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, + pointer: *const A, other: u8, tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. @@ -278,6 +288,7 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, + pointer: *const B, other: u8, tag: [*:0]const u8, static_val: u8, @@ -291,6 +302,7 @@ test mapAlloc { .other = 7, .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -298,6 +310,7 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); + defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); @@ -309,6 +322,7 @@ test mapAlloc { try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); + try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: @@ -332,13 +346,6 @@ pub fn MapRestrict(From: type, To: type) type { if (!Contains(T, From)) return void; - // `visit` walks a bounded array's `len` elements, but its mapped - // struct would hold the whole buffer: pairing the two by visit - // order would miss or misplace elements. - if (@typeInfo(T) == .@"struct" and @hasDecl(T, "constSlice") and @hasDecl(T, "slice")) { - stdx.debug.compileError("zml.meta.MapRestrict({any}, {any}) cannot map the bounded array {any}: use a slice or an array", .{ From, To, T }); - } - return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. diff --git a/zml/nn.zig b/zml/nn.zig index e8d9873d..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,11 +2,13 @@ const std = @import("std"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -31,6 +33,10 @@ pub const Linear = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { + zml.Buffer.deinitAll(Linear, self); + } + pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { return if (self.quantization) |q| q.scheme else null; } @@ -222,8 +228,8 @@ test "block128 scaled dot layouts" { const io = std.testing.io; const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) - if (platform.target != .cuda) return error.SkipZigTest; - const dtype = Quantization.Target.of(platform).fp8.?; + if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; + const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; @@ -316,13 +322,23 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - return ops.customCall( - "zml$scaled_dot_unmatched", - .{ in[0], in[1], in[2], in[3] }, - out_shape, - {}, - .{ .has_side_effect = false }, - ); + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { @@ -2350,8 +2366,8 @@ pub fn softplus( // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. const z = x.exp(); const u = z.addConstant(1); - const log1p = u.cmp(.EQ, .scalar(1, .f32)).select(z, z.mul(u.log()).div(u.subConstant(1))); - return x.cmp(.GT, .scalar(opts.threshold, .f32)).select(x, log1p); + const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); } test "softplus" { @@ -2361,14 +2377,11 @@ test "softplus" { const platform = zml.testing.env(); const input: Tensor = .init(.{ .x = inputs.len }, .f32); const Local = struct { - fn call(x: Tensor) Tensor { - return softplus(x, .{ .threshold = 2.0 }); + fn call(x: Tensor, dtype: DataType) Tensor { + return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); } }; - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{input}, .{}); - defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes( std.testing.io, platform, @@ -2377,13 +2390,18 @@ test "softplus" { ); defer input_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer result.deinit(); + for ([_]DataType{ .f32, .bf16 }) |dtype| { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); + defer exe.deinit(); - try zml.testing.expectClose( - std.testing.io, - Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), - result, - .{}, - ); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); + } } diff --git a/zml/ops.zig b/zml/ops.zig index b1a26d23..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -8,13 +8,6 @@ const platforms = @import("platforms"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; - -test { - _ = manual_computation; -} - const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -23,15 +16,113 @@ const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; + +test { + _ = manual_computation; +} + const log = std.log.scoped(.@"zml/Compiler"); +/// Reduces local tensor values across all processes in the computation. +/// For each tensor index, `func_` combines the values at that index from every +/// process. The result is identical on all processes. +/// +/// Inside a manual computation, `inputs` describes the local tensors at each +/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives +/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. +/// +/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. +/// Each input must have matching shapes and dtypes across processes. +/// * `func_` describes how to combine two scalar contributions: +/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such +/// as `Tensor.add`. This also applies to a one-element tuple or array. +/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a +/// tuple or array with one scalar tensor per input, in the same order. +/// +/// Returns the same container type as `inputs`, preserving tensor attributes. +/// With one process, returns `inputs` unchanged. +/// +/// The reduction uses one group containing all processes, including mesh replicas. +/// To reduce along one logical mesh axis, use `allReduceOnAxis`. pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const count = Compiler.current().meshes[0].numDevices(); + var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); + return allReduceGroups(inputs, func_, devices[0..count], count); +} + +/// Reduces local tensor values within process groups defined by `axis` in `mesh`. +/// A group contains processes with the same coordinates on every other logical +/// mesh axis. Separate replicas of the mesh belong to separate groups. Within +/// each group, `func_` combines corresponding elements and the result is identical +/// on all member processes. +/// +/// Use this inside a manual computation to combine partial results along `axis` +/// while preserving the independent computations along the other mesh axes. +/// +/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. +/// * `mesh` is the logical mesh used by the manual computation and must be +/// included in the compilation's meshes. +/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum +/// literal or a mesh axis tag. The axis must exist in `mesh`. +/// +/// Returns the same container type with tensor attributes as +/// `inputs`. An axis of size one returns `inputs` unchanged. +/// +/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local +/// expert contributions independently for each of the two data shards: +/// +/// ```zig +/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); +/// ``` +/// +/// Each expert shard now has the complete output for its data shard. To expose +/// this result through `manualComputation`, an output spec such as +/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the +/// result replicated along `.experts`. +pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); + const size = mesh.grid.axis_sizes[logical_axis]; + var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + const replicas = mesh.numReplicas(); + for (0..mesh.numDevices()) |device| { + const coords = mesh.grid.logical_coords[device]; + var tile: usize = 0; + var group: usize = 0; + for (0..mesh.logical.axes.len) |dim| { + tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; + if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; + } + group = group * replicas + replica_indices[tile]; + replica_indices[tile] += 1; + groups[group * size + coords[logical_axis]] = @intCast(device); + } + return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); +} + +/// Reduces corresponding tensor elements independently within each process group. +/// `inputs`, `func_` and the return value follow `allReduce`'s contract. +/// +/// `groups` identifies processes by their executable partition IDs, as returned +/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. +/// +/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions +/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group +/// receive that group's result. Groups of size one return `inputs` unchanged. +fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -75,8 +166,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.meshes[0].numDevices(); - if (num_devices <= 1) return inputs; + if (group_size <= 1) return inputs; const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); @@ -125,14 +215,9 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ break :b block; }; - var replica_group_storage: [constants.MAX_RANK]i64 = undefined; - for (0..@intCast(num_devices)) |i| { - replica_group_storage[i] = @intCast(i); - } - const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), - replica_group_slice, + .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), + groups, ); const handle = ctx.nextChannelId(); @@ -202,6 +287,20 @@ fn AllReduceReturnType(comptime InputsT: type) type { }; } +/// Returns the current process's zero based coordinate on `axis` in `mesh` +/// as a scalar u32. +/// +/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates +/// 0 and 1. Use this coordinate inside a manual computation to identify a shard +/// along one axis while `partitionId()` identifies a partition across the whole mesh. +pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); + var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; + for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; + return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) + .gather(.{ .device = partitionId() }, .{}); +} + pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); @@ -830,10 +929,9 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and mesh info over + // Copy tags; the partitioner infers result sharding. // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } @@ -1434,7 +1532,15 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, + symmetric_workspace: ?SymmetricWorkspace = null, output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + + /// Named per-device workspace shared across CuTe calls and executables. + /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. + pub const SymmetricWorkspace = struct { + name: []const u8, + bytes: i64, + }; }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1455,7 +1561,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1480,6 +1586,13 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); } + if (opts.symmetric_workspace) |workspace| { + stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); + attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), + .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), + }))); + } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = stablehlo.custom_call( @@ -2647,11 +2760,6 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2685,23 +2793,7 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2729,44 +2821,6 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } -pub fn shardingAwareTypedCustomCall( - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - - const Handler = struct { - input: Input, - attributes: Attributes, - - fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); - } - }; - const output_tensors = manualComputation( - Handler.body, - .{ .input = input, .attributes = attributes }, - @as([]const Shape, &output_shapes), - ); - - // Convert the slice back to a struct - var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; - } - return out; -} - pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2781,7 +2835,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2791,7 +2845,6 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2807,7 +2860,6 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2872,7 +2924,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + if (ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2884,7 +2936,6 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -2937,7 +2988,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2948,9 +2999,17 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index c9596f98..a0ec0037 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,49 +1,131 @@ const std = @import("std"); -const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const Compiler = @import("../Compiler.zig"); -const DataType = @import("../dtype.zig").DataType; +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; + const log = std.log.scoped(.@"zml/Compiler"); -/// ZML operator describes computation across several devices, -/// but `manualComputation` allows to partition the devices in small group, -/// split the `Tensor` on those partitions, -/// and describes sub-computation that work inside a partition. +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. /// -/// See: -/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. /// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: -/// - `I` is user defined and contains inputs tensors and any metadata required by the computation -/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs -/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs -/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated -/// * `outputs` contains the global output shapes, -/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` /// -/// This depart from the rest of ZML where output shapes are generally inferred, -/// but in this case since `body_fn` explains how to compute parts of the result, -/// it's not clear how to put them back together, -/// hence why the user is asked to explicitly provided the expected output shape. +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -54,13 +136,42 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - var owned_inputs: Inputs = inputs; - const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - defer arena.free(flat_input_tensors); + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); - var owned_output_shapes: OutputShapes = output_shapes; - const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); - defer arena.free(flat_output_shapes); + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } const WrappedInputs = struct { inputs: *const Inputs, @@ -72,15 +183,17 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + _ = manualComputationInternal( compiler, arena, + mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -90,6 +203,7 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; + return outputs_struct; } @@ -100,27 +214,21 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []*Shape, + global_output_shapes: []const Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { - .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), - }; - const manual_axes = collectManualAxes(mesh, input_tensors); - log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - const global_output_shapes = try arena.alloc(Shape, output_shapes.len); - for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -134,11 +242,17 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = mlir.Block.init(input_types, input_locs); - errdefer manual_block.deinit(); + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -150,17 +264,14 @@ noinline fn manualComputationInternal( }; } - for (output_shapes, global_output_shapes) |shape_ptr, global| { - shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { - stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -214,97 +325,14 @@ noinline fn manualComputationInternal( }, }; - // reuse local_outputs buffer to store the final result. + // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; } return local_outputs; } -const ChosenMesh = union(enum) { - mesh: *const Sharding.Mesh, - incompatible: [2]u32, -}; - -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { - var mesh: ?*const Sharding.Mesh = null; - var mesh_source_id: u32 = @intCast(inputs.len); - for (0.., inputs) |i, input| { - if (input.shape()._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) - return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; - } else { - mesh = shape_mesh; - mesh_source_id = @intCast(i); - } - } - } - return .{ .mesh = mesh orelse fallback }; -} - -test chooseCommonMesh { - const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); - const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); - const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); - const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); -} - -fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { - var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); - for (inputs) |input| { - const sharding = input.shape()._sharding; - if (sharding.mesh == null) continue; - std.debug.assert(sharding.mesh.? == mesh); - for (0..input.rank()) |dim| { - if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; - } - } - - var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; - for (mesh.logical.axes.constSlice(), 0..) |axis, i| { - if (used[i]) axes.appendAssumeCapacity(axis); - } - return axes; -} - -test collectManualAxes { - const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); - var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); - var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); - - const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); - data = replicated; - model = replicated; - try std.testing.expectEqual(@as(usize, 2), axes.len); - try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); - try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); - - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const expected = shardy.ManualAxesAttr.get(ctx, &.{ - .init(ctx, "data"), - .init(ctx, "model"), - }).attribute(); - try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); -} - const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -347,54 +375,172 @@ const gspmd = struct { } }; -const shardy_ = struct { - pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { - var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; - for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); - } -}; - -test "manualComputation passes local output shapes to the callback" { +test manualComputation { + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); - const partitions = mesh.numPartitionsForLogicalAxis(.model); - if (partitions == 1) return error.SkipZigTest; - var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); - defer compiler.deinit(); - compiler.activate(); - defer compiler.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - const scope = compiler.pushBlock(block); - defer scope.pop(); - - const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) - .withPartitioning(mesh, .{ .h = .model }); - const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); - const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); - var received_shape: ?Shape = null; - const Context = struct { - input: Tensor, - received_shape: *?Shape, + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, fn body(self: @This(), output_shape: Shape) Tensor { - self.received_shape.* = output_shape; - return self.input; + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); } }; - const output = manualComputation( - Context.body, - .{ .input = input, .received_shape = &received_shape }, - global_shape, - ); - try std.testing.expect(received_shape != null); - try std.testing.expect(received_shape.?.eql(local_shape)); - try std.testing.expect(output.shape().eql(global_shape)); - try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bfa4fc7d..824d2bd1 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -14,11 +14,7 @@ const log = std.log.scoped(.@"zml/pjrtx"); pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; - /// Compiles on the calling thread: under zio, `Compiler.compile` has - /// already moved off the coroutine onto a pool worker of its own - /// (`zio.blockInPlaceReserved`), and XLA's compile needs the thread's - /// stack, not a task's. - pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); @@ -41,7 +37,7 @@ pub const Client = opaque { }; }; - return @ptrCast(try client.compileRaw(api, .{ + return @ptrCast(try client.compile(api, io, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, diff --git a/zml/platform.zig b/zml/platform.zig index 6f9d114d..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,12 +3,13 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_rocm = @import("platforms/rocm"); const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -38,10 +39,6 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); return error.MissingDevices; } - if (num_devices > Platform.MAX_NUM_DEVICES) { - log.err("Platform {} got {} devices, but ZML supports up to {}", .{ target, num_devices, Platform.MAX_NUM_DEVICES }); - return error.InvalidDeviceCount; - } switch (target) { .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { if (!std.math.isPowerOfTwo(num_devices)) { @@ -262,12 +259,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, fn deinit(self: *CudaState) void { - if (self.fi_cutlass_moe_runners) |runners| { + if (self.flashinfer_cutlass_moe_runners) |runners| { runners.deinit(); - self.fi_cutlass_moe_runners = null; + self.flashinfer_cutlass_moe_runners = null; } } }; @@ -305,16 +302,6 @@ pub const Platform = struct { meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), replicated_mesh: *const zml.Mesh, io_impl: IoImpl, - /// Whether `releaseFreedDeviceMemory` asks the devices to give freed - /// memory back. `init` sets it for XLA's VMM allocator on CUDA and ROCm - /// (`CreateOptions.XlaGpu.Allocator.vmm`), which keeps freed memory - /// mapped, and counted in use, until it is reused or reclaimed. Null - /// otherwise: other allocators free at once, and oneAPI's VMM allocator - /// is not proven to follow the same rule. The flag lives in the - /// platform's arena, not in `Platform`: the first device that does not - /// answer as the VMM allocator does clears it for good, through a - /// `*const Platform`, without writing to the platform. - releases_freed_memory: ?*std.atomic.Value(bool) = null, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -328,9 +315,11 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); - errdefer pjrt_client.deinit(api); const pjrt_devices = pjrt_client.addressableDevices(api); try validateDeviceCount(target, pjrt_devices.len); + if (pjrt_devices.len > MAX_NUM_DEVICES) { + log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); + } const pjrt_memories = pjrt_client.addressableMemories(api); @@ -364,18 +353,7 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); - errdefer if (comptime Target.cuda.isEnabled()) platform.state.deinit(); try platform.meshes.ensureTotalCapacity(arena, 8); - const keeps_freed_memory = switch (target) { - .cuda => options.cuda.allocator == .vmm, - .rocm => options.rocm.allocator == .vmm, - else => false, - }; - if (keeps_freed_memory) { - const releases = try arena.create(std.atomic.Value(bool)); - releases.* = .init(true); - platform.releases_freed_memory = releases; - } { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -405,16 +383,9 @@ pub const Platform = struct { switch (target) { .cuda => { - if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { - zml.moe.cutlass_flashinfer.register(platform) catch |err| { - log.warn( - "Failed to register FlashInfer CUTLASS MoE custom calls: {}", - .{err}, - ); - }; - } else |err| { - log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); - } + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + }; }, else => {}, } @@ -448,117 +419,6 @@ pub const Platform = struct { } else error.Unavailable; } - /// Forces lazy device allocators to materialize their backing pools. - /// - /// In particular, a preallocated GPU BFC allocator does not reserve its - /// fixed arena until its first non-empty allocation. The temporary buffer - /// is released immediately; caching allocators retain their initialized - /// pools for subsequent allocations. Calling this more than once is safe. - /// Devices are warmed concurrently: a serial pass over eight GPUs costs - /// measurable milliseconds ahead of a load. A device whose task `io` - /// cannot spawn is warmed in place. - pub fn warmupDeviceAllocators(self: *const Platform, io: std.Io) !void { - const Worker = struct { - platform: *const Platform, - device_index: usize, - first_error: *stdx.FirstError(pjrt.ApiError), - - fn run(worker: @This()) void { - const dims: []const i64 = &.{1}; - const memory = worker.platform.devices[worker.device_index].memory(.default).?; - const buffer = worker.platform.pjrt_client.createUninitializedBuffer(worker.platform.pjrt_api, .{ - .dims = dims, - .element_type = pjrtx.bufferTypeFromDtype(.u8), - .layout = worker.platform.defaultMemoryLayout(dims, .u8), - .dst = .{ .memory = memory.pjrt_memory }, - }) catch |err| { - _ = worker.first_error.set(err); - return; - }; - buffer.deinit(worker.platform.pjrt_api); - } - }; - var first_error: stdx.FirstError(pjrt.ApiError) = .{}; - var group: std.Io.Group = .init; - for (self.devices, 0..) |_, device_index| { - const worker: Worker = .{ .platform = self, .device_index = device_index, .first_error = &first_error }; - group.concurrent(io, Worker.run, .{worker}) catch worker.run(); - } - try group.await(io); - if (first_error.get()) |err| return err; - } - - /// Gives back to the devices memory that nothing uses but that stays - /// held: the freed buffers XLA's VMM allocator keeps, which - /// `Device.memoryStats` counts in `bytes_in_use`, and on ROCm what the - /// HIP runtime keeps. - /// - /// XLA's VMM allocator keeps a freed allocation mapped, and counted in - /// use, until a request of the same size reuses it, a request beyond - /// the device's limit reclaims it, or an executable with remapped - /// command buffers is destroyed. A request one byte beyond the limit - /// completes the deferred frees of the device's memory, once its - /// compute stream reached them, then fails with `ResourceExhausted`, as - /// expected. It leaves out the frees of collective memory, which stay - /// counted, and of pinned host memory, which `bytes_in_use` does not - /// count. Any other answer means the plugin does not follow that rule: - /// it is logged once, naming the device, and clears - /// `releases_freed_memory`, so later calls skip this part. This part is - /// a no-op for other allocators, which free at once, and for a device - /// that reports no limit. - /// - /// On ROCm, whatever the allocator, it first makes the HIP runtime give - /// back the device memory it keeps outside the allocator, chiefly the - /// scratch memory of kernels that spill registers, which `memoryStats` - /// never counts but the devices do - /// (`platforms_rocm.releaseRuntimeDeviceMemory`). That part reaches - /// every queue the process has on these devices, other platforms' - /// included, and makes running work wait: each queue holding scratch is - /// suspended and resumed, the call waits for its kernels using scratch, - /// and allocations on the device stall meanwhile. A call within a load - /// can overlap the load's other compiles and transfers, which then wait - /// too. A missing HIP runtime, or a device that does not refuse, is - /// logged once and ends that part for the process. - pub fn releaseFreedDeviceMemory(self: *const Platform) void { - if (comptime Target.rocm.isEnabled()) { - if (self.target == .rocm) _ = self.releaseRocmRuntimeMemory(); - } - const releases = self.releases_freed_memory orelse return; - for (self.devices) |*device| { - if (!releases.load(.monotonic)) return; - const limit = device.memoryStats().bytes_limit orelse continue; - // A limit that large is no limit: nothing goes beyond it. - const beyond_limit = std.math.cast(i64, limit + 1) orelse continue; - const dims: []const i64 = &.{beyond_limit}; - const answer: []const u8 = if (self.pjrt_client.createUninitializedBufferSilent(self.pjrt_api, .{ - .dims = dims, - .element_type = pjrtx.bufferTypeFromDtype(.u8), - .layout = self.defaultMemoryLayout(dims, .u8), - .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, - })) |buffer| answer: { - buffer.deinit(self.pjrt_api); - break :answer "a buffer"; - } else |err| switch (err) { - error.ResourceExhausted => continue, - else => @errorName(err), - }; - if (releases.swap(false, .monotonic)) { - log.warn("Device {d} answered a request beyond its memory limit with {s}, not ResourceExhausted; freed device memory is no longer given back and may stay counted as in use", .{ device.id(), answer }); - } - return; - } - } - - /// `platforms_rocm.releaseRuntimeDeviceMemory` over every device: a - /// device's HIP ordinal is its local hardware id. - fn releaseRocmRuntimeMemory(self: *const Platform) bool { - var ordinals: [MAX_NUM_DEVICES]c_int = undefined; - for (self.devices, ordinals[0..self.devices.len]) |*device, *ordinal| { - ordinal.* = device.localHardwareId(); - } - return platforms_rocm.releaseRuntimeDeviceMemory(ordinals[0..self.devices.len]); - } - pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; @@ -738,14 +598,6 @@ pub const Platform = struct { try writer.writeAll(" }"); } - /// `shape` placed as a buffer places it: on its mesh, or replicated on - /// every device (`replicated_mesh`) when it names none. - pub fn placedShape(self: *const Platform, shape: zml.Shape) zml.Shape { - var placed = shape; - if (placed._sharding.mesh == null) placed._sharding.mesh = self.replicated_mesh; - return placed; - } - pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { for (self.memories) |mem| { if (mem.isOfKind(kind)) { @@ -1008,6 +860,12 @@ pub const cuda = struct { } }; + /// SM count of an addressable CUDA device; unavailable attributes return null. + pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { + const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; + return std.math.cast(u32, attribute.int64); + } + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { if (platform.target != .cuda) return null; const devices = platform.pjrt_client.devices(platform.pjrt_api); @@ -1020,6 +878,29 @@ pub const cuda = struct { } } else null; } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } }; pub const rocm = struct { @@ -1134,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1163,9 +1047,13 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); @@ -1207,63 +1095,3 @@ test "platform defaultMemoryLayout is boring" { }); } } - -test "platform device allocators can be warmed repeatedly" { - const platform = zml.testing.env(); - - try platform.warmupDeviceAllocators(std.testing.io); - try platform.warmupDeviceAllocators(std.testing.io); -} - -test "platform ROCm devices refuse a request for their whole memory, then still work" { - if (comptime !Target.rocm.isEnabled()) return error.SkipZigTest; - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - // Every device refuses a request for its whole memory, the request ROCr - // answers by reclaiming its queues' scratch and freeing its caches. The - // release itself is not visible from here: HIP counts ROCr's caches as - // free, and only the driver's counters in sysfs show it. - try std.testing.expect(platform.releaseRocmRuntimeMemory()); - - // The devices still take and give back data afterwards. - const values = [_]u32{ 1, 2, 3, 4, 5, 6, 7, 8 }; - var buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, .init(.{ .n = values.len }, .u32), std.mem.sliceAsBytes(&values)); - defer buffer.deinit(); - const read: zml.Slice = try buffer.toSliceAlloc(std.testing.allocator, std.testing.io); - defer read.free(std.testing.allocator); - try std.testing.expectEqualSlices(u32, &values, read.constItems(u32)); -} - -test "platform gives freed device memory back" { - const platform = zml.testing.env(); - // On GPU CI, CUDA and ROCm run this with XLA's VMM allocator. - const releases = platform.releases_freed_memory orelse return error.SkipZigTest; - const device = &platform.devices[0]; - // Without a limit the drain does nothing. - try std.testing.expect(device.memoryStats().bytes_limit != null); - - // An odd size, so that no request of the same size reuses it once freed. - const size = 64 * 1024 * 1024 + 4096; - const dims: []const i64 = &.{size}; - const buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, .{ - .dims = dims, - .element_type = pjrtx.bufferTypeFromDtype(.u8), - .layout = platform.defaultMemoryLayout(dims, .u8), - .dst = .{ .memory = device.memory(.default).?.pjrt_memory }, - }); - { - // Once its allocation is ready, destroying it frees it at once, - // not later on another thread. - errdefer buffer.deinit(platform.pjrt_api); - const ready = buffer.readyEvent(platform.pjrt_api); - defer ready.deinit(platform.pjrt_api); - try ready.await(platform.pjrt_api, std.testing.io); - } - buffer.deinit(platform.pjrt_api); - const in_use_freed = device.memoryStats().bytes_in_use; - - platform.releaseFreedDeviceMemory(); - try std.testing.expect(releases.load(.monotonic)); - try std.testing.expect(device.memoryStats().bytes_in_use + size <= in_use_freed); -} diff --git a/zml/quantization.zig b/zml/quantization.zig index 9053b26c..db20619a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -105,38 +105,6 @@ pub const Quantization = struct { return self == .mxfp8 or self == .mxfp4; } }; - - /// What a platform's FP8 GEMMs read, so block-FP8 weights can be bound - /// for it once. Binding them requires the store to carry one - /// (`zml.io.TensorStore.Options.quantization`); the code that binds them - /// reads it (`TensorStore.View.quantization`) and converts or lays them - /// out while binding. - pub const Target = struct { - /// The E4M3 encoding of FP8 weights and activations: FNUZ on CDNA3, - /// FN on CUDA, CDNA4 and RDNA4. Null on other platforms: block-FP8 - /// activations stay BF16 (`quantizeInput`), and weights keep the - /// checkpoint's encoding. - fp8: ?DataType = .f8e4m3fn, - /// Whether e8m0 block scales are laid out in the 128x4 blocks of - /// Blackwell's block-scaled MMA (CUDA sm_10x). - swizzled_scales: bool = false, - - /// `platform`'s target: on CUDA, FN, its scales swizzled on sm_10x; - /// on ROCm, by architecture, null when its compute capability is - /// unknown; elsewhere, CPU included, null and unswizzled. The only - /// platform switch for block-FP8 weights. - pub fn of(platform: *const Platform) Target { - return switch (platform.target) { - .cuda => .{ .swizzled_scales = if (platform_mod.cuda.computeCapability(platform)) |cc| cc.major == 10 else false }, - .rocm => .{ .fp8 = if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null }, - else => .{ .fp8 = null }, - }; - } - }; }; pub const QuantizedInput = struct { @@ -159,7 +127,18 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, .fp8_block32, .fp8_block128 => blk: { - const dtype = Quantization.Target.of(platform).fp8 orelse return null; + const dtype: DataType = switch (platform.target) { + .cuda => .f8e4m3fn, + // RNGD contracts matching FP8 pairs in 128-wide K blocks. + .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, + } else return null, + else => return null, + }; + break :blk switch (quantization.scheme) { .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), @@ -187,6 +166,23 @@ pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { }; } +pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { + const platform = zml.Compiler.current().platform; + + const dtype: ?DataType = switch (platform.target) { + .cuda, .cpu, .tpu => .f8e4m3fn, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null, + else => null, + }; + + if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); + return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); +} + fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { const bits = raw_scale.bitCast(.u32); return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index beee2cca..3827fe27 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,19 +2,497 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const testing = @import("testing.zig"); - -pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); const log = std.log.scoped(.@"zml/safetensors"); -const BYTES_HEADER = 8; +// TODO: consider promoting this to the top level struct +// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights +/// Maps tensor names to their shape and positions in different sub files. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: std.StringArrayHashMapUnmanaged(Tensor), + + /// Metadata found in `__metadata__` field of the safetensors header. + /// This is rarely use and the information there is not standardized, so we let user code handle it. + /// Here we just store the plain bytes, reach out if you need this to be already parsed. + metadata: []const u8, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = "", + }; + } + + pub fn deinit(self: *TensorRegistry) void { + self.arena.deinit(); + } + + pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { + const file_type, const file = try resolveModelEntrypoint(io, repo); + defer file.close(io); + + var registry: TensorRegistry = .init(allocator); + switch (file_type) { + .safetensors => try registry.parseSafetensors(io, allocator, file), + .index => try registry.parseIndexJson(io, allocator, repo, file), + .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), + .unknown => return error.InvalidPath, + } + return registry; + } + + pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { + const file_type: FileType = .resolve(path); + switch (file_type) { + .safetensors => { + // open directly + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + var registry: TensorRegistry = .init(allocator); + try registry.parseSafetensors(io, allocator, file); + return registry; + }, + .full_index, + .index, + => { + // resolve repo as parent folder then open + const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + + var registry: TensorRegistry = .init(allocator); + try switch (file_type) { + .index => registry.parseIndexJson(io, allocator, repo, file), + .full_index => registry.parseFullIndexJson(io, allocator, repo, file), + .safetensors, .unknown => unreachable, + }; + return registry; + }, + .unknown => { + // the path is possibly a folder, let's try that + const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return try fromRepo(allocator, io, repo); + }, + } + } + + /// Register a tensor, duplicate tensor name and uri. + /// If you create a lot of tensors sharing the same uri, + /// considere calling `registry.tensors` directly and avoid duplicated uris. + pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { + const allocator = self.arena.allocator(); + + const owned_name = try allocator.dupe(u8, tensor.name); + errdefer allocator.free(owned_name); + const owned_uri = try allocator.dupe(u8, tensor.file_uri); + errdefer allocator.free(owned_uri); + + const owned_tensor: Tensor = .{ + .name = owned_name, + .file_uri = owned_uri, + .shape = tensor.shape, + .offset = tensor.offset, + }; + try self.tensors.put(allocator, owned_name, owned_tensor); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } + + /// Parse a given .safetensors file and create entries for each tensor + pub fn parseSafetensors( + self: *TensorRegistry, + io: std.Io, + scratch_allocator: std.mem.Allocator, + file: std.Io.File, + ) !void { + var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); + defer tmp_state.deinit(); + const tmp_arena = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); + const file_name_len = try file.realPath(io, owned_file_name); + // Try to release extra memory, but we don't care if it fails + _ = self_arena.resize(owned_file_name, file_name_len); + owned_file_name = owned_file_name[0..file_name_len]; + + const header_buf = try tmp_arena.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = file.reader(io, header_buf); + + const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); + const BYTES_HEADER = 8; + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); + + try file_reader.interface.readSliceAll(json_data); + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + tmp_arena, + json_data, + .{}, + ); + + try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); + for (headers.object.keys(), headers.object.values()) |key, value| { + if (std.mem.eql(u8, key, "__metadata__")) { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + continue; + } + + const owned_name = try self_arena.dupe(u8, key); + const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); + self.tensors.putAssumeCapacity(owned_name, tensor); + } + } + + pub fn parseIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; + try safetensors_files.ensureUnusedCapacity(tmp, 64); + if (index.object.get("weight_map")) |wm| { + for (wm.object.values()) |file_name| { + try safetensors_files.put(tmp, file_name.string, {}); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + for (safetensors_files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try self.parseSafetensors(io, tmp, file); + } + } + + pub fn parseFullIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + + var repo_uri_buf = try tmp.alloc(u8, 1024); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + + // Note: this is relatively inefficient, we read the full file, copy most of it's content, + // then reduce to TensorRegistry format. + // We could update TensorRegistry as we stream the json. + // Zig std json API are a bit awkward around streaming though. + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try realpaths.ensureUnusedCapacity(tmp, 64); + const wm = index.object.get("weight_map") orelse return error.InvalidJson; + + try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); + for (wm.object.keys(), wm.object.values()) |key, value| { + const tensor_name = try self_arena.dupe(u8, key); + const object = value.object; + const filename = value.object.get("filename").?.string; + const filename_entry = try realpaths.getOrPut(tmp, filename); + const file_uri = if (filename_entry.found_existing) + filename_entry.value_ptr.* + else uri: { + const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); + filename_entry.value_ptr.* = uri; + break :uri uri; + }; + + const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); + self.tensors.putAssumeCapacity(tensor_name, tensor_info); + } + + if (index.object.get("metadata")) |metadata| { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + } + } + + pub fn saveAsFullIndexJson( + self: *const TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + out: *std.Io.Writer, + ) !void { + var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); + defer gpa.free(repo_uri_buf); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + var total_size: u64 = 0; + for (self.tensors.values()) |value| total_size += value.byteSize(); + + try out.print( + \\{{ + \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, + \\"weight_map": {{ + \\ + , .{ self.tensors.count(), total_size }); + + const n = self.tensors.count(); + for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { + std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); + const local_name = value.file_uri[repo_name.len + 1 ..]; + const end_offset = value.offset + value.byteSize(); + const safetensors_dtype: []const u8 = dt: { + for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { + if (zml_dt == value.shape._dtype) break :dt st_dt; + } + // Note: we could bitcast to u8, + // but reaching this panic implies the user didn't loaded this TensorRegistry + // from a safetensors file, and manually created the dict. + // In which case, they know what they are doing, + // and can easily insert the bitcast themselves. + std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); + }; + const comma = if (i + 1 == n) "" else ","; + try out.print( + \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} + \\ + , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); + } + + try out.writeAll( + \\} + \\} + \\ + ); + try out.flush(); + } +}; + +pub const TensorReader = struct { + tensor: Tensor, + file: std.Io.File, + file_reader: std.Io.File.Reader, + alignment: ?std.mem.Alignment, + remaining: u64, + padding_remaining: u64, + + io: std.Io, + interface: std.Io.Reader, + + pub const InitOpts = struct { + alignment: ?std.mem.Alignment = null, + }; + + pub fn init( + io: std.Io, + tensor: Tensor, + buffer: []u8, + opts: InitOpts, + ) !TensorReader { + const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); + errdefer file.close(io); + + const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { + const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); + break :blk .{ aligned_offset, tensor.offset - aligned_offset }; + } else blk: { + break :blk .{ tensor.offset, 0 }; + }; + + var file_reader = file.reader(io, buffer); + try file_reader.seekTo(offset); + + return .{ + .tensor = tensor, + .file = file, + .file_reader = file_reader, + .alignment = opts.alignment, + .remaining = tensor.byteSize(), + .padding_remaining = padding_remaining, + .io = io, + .interface = .{ + .vtable = if (opts.alignment) |_| blk: { + break :blk &.{ + .stream = streamAligned, + .discard = discardAligned, + }; + } else blk: { + break :blk &.{ + .stream = stream, + .discard = discard, + }; + }, + .buffer = &.{}, + .seek = 0, + .end = 0, + }, + }; + } + + pub fn deinit(self: *TensorReader) void { + self.file.close(self.io); + } + + // Limit single reads to 1GB to avoid issues on macOS where pread returns + // INT_MAX when the requested size is larger than 2GB + const max_single_read_bytes: u64 = 1 << 30; + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); + const n = try self.file_reader.interface.stream(w, combined_limit); + self.remaining -= n; + return n; + } + + fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } + + fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const align_bytes = self.alignment.?.toByteUnits(); + const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); + const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; + const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); + + if (file_offset != writer_offset) { + const skip = if (file_offset >= writer_offset) blk: { + break :blk file_offset - writer_offset; + } else blk: { + break :blk align_bytes - writer_offset + file_offset; + }; + + w.advance(skip); + } + + if (w.writableSliceGreedy(align_bytes)) |dest| { + const phys_buf = dest.ptr - file_offset; + const phys_pos = self.file_reader.pos - file_offset; + const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); + + if (phys_size > file_offset) { + const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; + if (phys_read > file_offset) { + const bytes_read = phys_read - file_offset; + w.advance(bytes_read); + self.file_reader.pos += bytes_read; + self.remaining -= bytes_read; + return bytes_read; + } + } + } else |_| {} + + const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); + self.remaining -= bytes_read; + + return bytes_read; + } + + fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) return 0; + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } +}; -/// A checkpoint tensor: its name, its shape, and where its bytes are, at -/// `offset` of the file `file_uri` names. The URI is what the io that read -/// the header resolved, a VFS URI under a VFS, so read the tensor through -/// an io that resolves it alike. A registry's entries own nothing; the -/// registry owns their strings. pub const Tensor = struct { file_uri: []const u8, name: []const u8, @@ -25,31 +503,32 @@ pub const Tensor = struct { return self.shape.byteSize(); } - /// Reads the whole tensor into `buffer`, which holds exactly its bytes - /// (`OutOfBounds` otherwise). Opens `file_uri` through `io` for the - /// call, failing as that io's open does; a file that ends inside the - /// tensor is `UnexpectedEndOfFile`. - pub fn read(self: Tensor, io: std.Io, buffer: []u8) !void { - if (buffer.len != self.byteSize()) return error.OutOfBounds; - const file = try std.Io.Dir.openFile(.cwd(), io, self.file_uri, .{ .mode = .read_only }); - defer file.close(io); - return self.readRange(io, file, buffer, 0); + pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { + return TensorReader.init(io, self, buffer, opts); } - pub const RangeError = std.Io.File.ReadPositionalError || error{ OutOfBounds, UnexpectedEndOfFile }; + pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { + const shape_field = object.get("shape").?.array; + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); + return error.TooManyDimensions; + } - /// Reads `buffer.len` bytes of the tensor from `tensor_offset`, relative - /// to its data, out of `file`, which the caller opened at `file_uri`. A - /// range past the tensor is `OutOfBounds`, a file that ends inside it - /// `UnexpectedEndOfFile`. The reads are positional, so concurrent calls - /// may share one file. - pub fn readRange(self: Tensor, io: std.Io, file: std.Io.File, buffer: []u8, tensor_offset: u64) RangeError!void { - const tensor_size = self.byteSize(); - if (tensor_offset > tensor_size or buffer.len > tensor_size - tensor_offset) return error.OutOfBounds; - if (buffer.len == 0) return; - const file_offset = std.math.add(u64, self.offset, tensor_offset) catch return error.OutOfBounds; - _ = std.math.add(u64, file_offset, buffer.len) catch return error.OutOfBounds; - if (try file.readPositionalAll(io, buffer, file_offset) != buffer.len) return error.UnexpectedEndOfFile; + const offset_field = object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(object.get("dtype").?.string); + + var dims: Shape.DimsArray = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + return .{ + .file_uri = file_uri, + .name = name, + .shape = .init(dims.slice(), dtype), + .offset = base_offset + start, + }; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -63,727 +542,100 @@ pub const Tensor = struct { } }; -test "Tensor.readRange reads tensor-relative ranges within the tensor and its file" { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); +pub const FileType = enum { + full_index, + index, + safetensors, + unknown, - const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); - defer file.close(io); - try file.writePositionalAll(io, "prefix0123456789", 0); - - var tensor: Tensor = .{ - .file_uri = "unused", - .name = "tensor", - .shape = .init(.{10}, .u8), - .offset = "prefix".len, - }; - var middle: [4]u8 = undefined; - try tensor.readRange(io, file, &middle, 3); - try std.testing.expectEqualStrings("3456", &middle); - - var empty: [0]u8 = .{}; - try tensor.readRange(io, file, &empty, 10); - try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &empty, 11)); - - var too_large: [2]u8 = undefined; - try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &too_large, 9)); - - // A file that ends inside the tensor. - tensor.shape = .init(.{12}, .u8); - var tail: [3]u8 = undefined; - try std.testing.expectError(error.UnexpectedEndOfFile, tensor.readRange(io, file, &tail, 9)); - - tensor.offset = std.math.maxInt(u64); - var overflow: [1]u8 = undefined; - try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 0)); - try std.testing.expectError(error.OutOfBounds, tensor.readRange(io, file, &overflow, 1)); -} - -test "Tensor.read reads the whole tensor from its file" { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const file = try tmp.dir.createFile(io, "tensor.bin", .{ .read = true }); - try file.writePositionalAll(io, "prefix0123456789", 0); - - var path_buffer: [1024]u8 = undefined; - const path_len = try file.realPath(io, &path_buffer); - file.close(io); - - const tensor: Tensor = .{ - .file_uri = path_buffer[0..path_len], - .name = "tensor", - .shape = .init(.{10}, .u8), - .offset = "prefix".len, - }; - var whole: [10]u8 = undefined; - try tensor.read(io, &whole); - try std.testing.expectEqualStrings("0123456789", &whole); - try std.testing.expectError(error.OutOfBounds, tensor.read(io, whole[0..9])); -} - -/// The tensors of one or more checkpoints by name, read from their -/// safetensors headers; no tensor data is read. It owns every name and URI. -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, - - tensors: Tensors, - - /// An empty registry. - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - }; - } - - /// The checkpoint in `repo`: its `model.safetensors.index.json` and the - /// files it lists, else `model.safetensors`. `repo` stays the caller's - /// to close. - pub fn fromRepo( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - ) !TensorRegistry { - const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); - defer entrypoint.close(io); - - var registry: TensorRegistry = .init(allocator); - errdefer registry.deinit(); - try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); - return registry; - } - - /// The checkpoint at `path`, read through `io`: a `.safetensors` file, - /// a `.safetensors.index.json` and the files it lists, or a directory - /// holding `model.safetensors.index.json` or `model.safetensors`. A - /// relative path is from the cwd; under a VFS, `path` may be a remote - /// URI. A missing path or entrypoint is `FileNotFound`, a denied - /// directory or entrypoint `PermissionDenied`; a file opened by name, - /// the one given or one an index lists, fails as `io`'s open does (a - /// denied local file is `AccessDenied`). A malformed header, such as - /// one whose offsets leave the file in a truncated download or one - /// shorter than its 8-byte length field, or an index without a - /// `weight_map` from tensor names to file names, is - /// `InvalidSafetensorsHeader`, and a name two of its files repeat - /// `DuplicateTensor`, both logged. - pub fn fromPath( - allocator: std.mem.Allocator, - io: std.Io, - path: []const u8, - ) !TensorRegistry { - const repo_path, const file_kind = locateCheckpoint(path); - var repo = try resolveModelRepo(io, repo_path); - defer repo.close(io); - const kind = file_kind orelse return fromRepo(allocator, io, repo); - - // `repo` is the file's directory: the file opens by its base name. - const entrypoint = try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }); - defer entrypoint.close(io); - - var registry: TensorRegistry = .init(allocator); - errdefer registry.deinit(); - try addCheckpoint(io, ®istry, repo, entrypoint, kind, ""); - return registry; - } - - /// Adds every tensor of the checkpoint in `repo`, read as `fromRepo` - /// reads it, under `name` and a dot: `View.withPrefix(name)` reaches - /// them. A second checkpoint, such as a speculative drafter's, joins - /// the registry under a name of its own, and one store then holds both. - /// Adding under a name that a tensor already here is named under fails - /// with `error.DuplicateTensor`, logged, before anything is read. `repo` - /// stays the caller's to close. - pub fn addRepo(self: *TensorRegistry, io: std.Io, repo: std.Io.Dir, name: []const u8) !void { - const prefix = try std.mem.concat(self.arena.child_allocator, u8, &.{ name, "." }); - defer self.arena.child_allocator.free(prefix); - // A checkpoint added beside another owns its prefix: no tensor already - // here may be named under it. - for (self.tensors.keys()) |key| { - if (std.mem.startsWith(u8, key, prefix)) { - log.warn("Cannot add a checkpoint under {s}: tensor {s} is already named under it", .{ name, key }); - return error.DuplicateTensor; - } + pub fn resolve(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { + return .full_index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; } - const entrypoint, const kind, _ = try resolveModelEntrypoint(io, repo); - defer entrypoint.close(io); - try addCheckpoint(io, self, repo, entrypoint, kind, prefix); - } - - /// Frees every entry, its name and URI. - pub fn deinit(self: *TensorRegistry) void { - const allocator = self.arena.allocator(); - self.tensors.deinit(allocator); - self.arena.deinit(); - } - - /// Adds `tensor`, its name and file URI copied; a name already here is - /// `DuplicateTensor`, logged. - pub fn registerTensor( - self: *TensorRegistry, - tensor: Tensor, - ) !void { - const allocator = self.arena.allocator(); - - var tensor_copy = tensor; - - tensor_copy.name = try allocator.dupe(u8, tensor.name); - tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - - const entry = try self.tensors.getOrPut(allocator, tensor_copy.name); - if (entry.found_existing) { - log.warn("Tensor {s} of {s} is already registered from {s}", .{ tensor.name, tensor.file_uri, entry.value_ptr.file_uri }); - return error.DuplicateTensor; - } - entry.value_ptr.* = tensor_copy; } }; -/// What a checkpoint's entrypoint is: an index of the safetensors files -/// holding its tensors, or the one file holding them. -const EntrypointKind = enum { index, safetensors }; - -/// Where the checkpoint at `path` is (`TensorRegistry.fromPath`): its -/// directory, and the kind of the file `path` names there, or null when -/// `path` names the directory itself. -fn locateCheckpoint(path: []const u8) struct { []const u8, ?EntrypointKind } { - const kind: ?EntrypointKind = if (std.mem.endsWith(u8, path, ".safetensors.index.json")) - .index - else if (std.mem.endsWith(u8, path, ".safetensors")) - .safetensors - else - null; - return .{ if (kind == null) path else std.fs.path.dirname(path) orelse ".", kind }; -} - -/// Adds the tensors of the checkpoint whose entrypoint is open in `repo` to -/// `registry` under `prefix`. An index's files are read from `repo`. -fn addCheckpoint( - io: std.Io, - registry: *TensorRegistry, - repo: std.Io.Dir, - entrypoint: std.Io.File, - kind: EntrypointKind, - prefix: []const u8, -) !void { - switch (kind) { - .safetensors => return parseSafetensors(io, registry, entrypoint, prefix), - .index => { - var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); - defer arena.deinit(); - const scratch = arena.allocator(); - - var index_uri_buf: [1024]u8 = undefined; - const index_uri = index_uri_buf[0..try entrypoint.realPath(io, &index_uri_buf)]; - - var index_reader = entrypoint.reader(io, try scratch.alloc(u8, 64 * 1024)); - const file_names = indexFiles(scratch, &index_reader.interface) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - error.ReadFailed => return index_reader.err orelse err, - else => { - log.warn("Invalid safetensors index {s}: needs a weight_map from tensor names to file names ({t})", .{ index_uri, err }); - return error.InvalidSafetensorsHeader; - }, - }; - for (file_names) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try parseSafetensors(io, registry, file, prefix); - } - }, - } -} - -/// The files a safetensors index maps tensors to, each once, in the order -/// its `weight_map` first names them, allocated in `arena`. An index -/// without a `weight_map` from tensor names to file names fails with the -/// JSON parser's error, unlogged. -fn indexFiles(arena: std.mem.Allocator, index: *std.Io.Reader) ![]const []const u8 { - var json_reader: std.json.Reader = .init(arena, index); - // The default allocation, `.alloc_always` over a `json.Reader`, copies - // every string out of `index`'s buffer, which the next refill overwrites. - const Index = struct { weight_map: std.json.ArrayHashMap([]const u8) }; - const parsed = try std.json.parseFromTokenSourceLeaky(Index, arena, &json_reader, .{ .ignore_unknown_fields = true }); - var files: std.StringArrayHashMapUnmanaged(void) = .empty; - for (parsed.weight_map.map.values()) |file_name| try files.put(arena, file_name, {}); - return files.keys(); -} - -/// The safetensors files `TensorRegistry.fromPath` reads for `path`, named -/// in their directory, which is returned open for the caller to close: the -/// file a `.safetensors` path names, the files a `.safetensors.index.json` -/// lists, else those the directory's `model.safetensors.index.json` lists, -/// or its `model.safetensors`. The names are allocated in `arena`. No -/// header is read and nothing is logged: a failure here is one `fromPath` -/// reports, with its context. -pub fn checkpointFiles(arena: std.mem.Allocator, io: std.Io, path: []const u8) !struct { std.Io.Dir, []const []const u8 } { - const repo_path, const file_kind = locateCheckpoint(path); - const repo = try std.Io.Dir.openDir(.cwd(), io, repo_path, .{}); - errdefer repo.close(io); - const entrypoint, const kind, const name = if (file_kind) |kind| - .{ try repo.openFile(io, std.fs.path.basename(path), .{ .mode = .read_only }), kind, std.fs.path.basename(path) } - else - try resolveModelEntrypoint(io, repo); - defer entrypoint.close(io); - const names = switch (kind) { - .safetensors => try arena.dupe([]const u8, &.{name}), - .index => names: { - var index_reader = entrypoint.reader(io, try arena.alloc(u8, 64 * 1024)); - break :names indexFiles(arena, &index_reader.interface) catch |err| switch (err) { - error.ReadFailed => return index_reader.err orelse err, - else => |e| return e, - }; - }, - }; - return .{ repo, names }; -} - -/// Adds the tensors of the safetensors `file` to `registry`, each named -/// `prefix` followed by its key in the header. A header that is not an -/// object of tensors, each with a `dtype`, a `shape` and `data_offsets` -/// spanning exactly its bytes within the file, is -/// `InvalidSafetensorsHeader`, and a name already registered -/// `DuplicateTensor`, both logged. -pub fn parseSafetensors( - io: std.Io, - registry: *TensorRegistry, - file: std.Io.File, - prefix: []const u8, -) !void { - var arena = std.heap.ArenaAllocator.init(registry.arena.child_allocator); - defer arena.deinit(); - - var reader = file.reader(io, try arena.allocator().alloc(u8, 64 * 1024)); - - var file_uri_buf: [1024]u8 = undefined; - const file_uri_len = try file.realPath(io, &file_uri_buf); - const file_uri = file_uri_buf[0..file_uri_len]; - - // `File.Reader` returns `ReadFailed` with its cause in `err`, which a - // failed seek leaves unset. - const json_header_length: u64 = reader.interface.takeInt(u64, .little) catch |err| switch (err) { - error.EndOfStream => { - log.warn("Invalid safetensors header in {s}: shorter than its {d}-byte header length", .{ file_uri, BYTES_HEADER }); - return error.InvalidSafetensorsHeader; - }, - error.ReadFailed => return reader.err orelse err, - }; - // The header and every tensor's data must lie inside the file. - const file_length = try file.length(io); - if (json_header_length > file_length -| BYTES_HEADER) { - log.warn("Invalid safetensors header in {s}: {d} bytes of header in a file of {d}", .{ file_uri, json_header_length, file_length }); - return error.InvalidSafetensorsHeader; - } - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - - reader.interface.readSliceAll(json_data) catch |err| return switch (err) { - error.ReadFailed => reader.err orelse err, - error.EndOfStream => err, - }; - - const headers = std.json.parseFromSliceLeaky(std.json.ArrayHashMap(std.json.Value), arena.allocator(), json_data, .{}) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - else => { - log.warn("Invalid safetensors header in {s}: not a JSON object ({t})", .{ file_uri, err }); - return error.InvalidSafetensorsHeader; - }, - }; - const data_len = file_length - data_start_offset; - - // `data_offsets` are `[start, end)` after the header. - const Entry = struct { dtype: []const u8, shape: []const i64, data_offsets: [2]u64 }; - var it = headers.map.iterator(); - while (it.next()) |kv| { - const key = kv.key_ptr.*; - - // Free-form strings, which nothing reads. - if (std.mem.eql(u8, key, "__metadata__")) continue; - - const entry = std.json.parseFromValueLeaky(Entry, arena.allocator(), kv.value_ptr.*, .{ .ignore_unknown_fields = true }) catch |err| switch (err) { - error.OutOfMemory => |e| return e, - else => { - log.warn("Invalid safetensors header in {s}: tensor {s} needs a string dtype, an integer shape and two unsigned data_offsets ({t})", .{ file_uri, key, err }); - return error.InvalidSafetensorsHeader; - }, - }; - - if (entry.shape.len > Shape.MAX_RANK) { - log.warn("Cannot load {s}: tensor {s} has {d} dims, more than {d}", .{ file_uri, key, entry.shape.len, Shape.MAX_RANK }); - return error.TooManyDimensions; - } - const dtype = stringToDtype(entry.dtype) orelse { - log.warn("Cannot load {s}: tensor {s} has dtype {s}, which zml does not support", .{ file_uri, key, entry.dtype }); - return error.UnsupportedDataType; - }; - - const start, const end = entry.data_offsets; - const byte_size = checkedByteSize(entry.shape, dtype); - if (byte_size == null or start > end or end > data_len or end - start != byte_size.?) { - log.warn("Invalid safetensors header in {s}: tensor {s} has data_offsets [{d},{d}], which must span the bytes of its {s} shape {any} within the file's {d} data bytes", .{ file_uri, key, start, end, entry.dtype, entry.shape, data_len }); - return error.InvalidSafetensorsHeader; - } - - const tensor: Tensor = .{ - .file_uri = file_uri, - .name = if (prefix.len == 0) key else try std.mem.concat(arena.allocator(), u8, &.{ prefix, key }), - .shape = .init(entry.shape, dtype), - .offset = data_start_offset + start, - }; - - try registry.registerTensor(tensor); - } -} - -/// The bytes of `dims` of `dtype`, as `Shape.byteSize` counts them, or null -/// on a negative dim or a bit count past u64. -fn checkedByteSize(dims: []const i64, dtype: DataType) ?u64 { - var bits: u64 = dtype.bitSizeOf(); - for (dims) |d| bits = std.math.mul(u64, bits, std.math.cast(u64, d) orelse return null) catch return null; - return std.math.divCeil(u64, bits, 8) catch unreachable; -} - -test "parseSafetensors rejects a malformed header, data_offsets that do not span their tensor within the file, or a name already registered" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const max = "18446744073709551615"; - // One U8 tensor over the file's 4 data bytes. `header` overrides the - // whole header, `header_len` the header's length field. - const cases = [_]struct { offsets: []const u8 = "[0,4]", shape: []const u8 = "[4]", header: ?[]const u8 = null, header_len: ?u64 = null, valid: bool = false }{ - .{ .valid = true }, - .{ .offsets = "[4,0]" }, - .{ .offsets = "[0,3]" }, - .{ .offsets = "[0,5]" }, - .{ .offsets = "[-4,4]" }, - .{ .offsets = "[0," ++ max ++ "]" }, - .{ .offsets = "[" ++ max ++ "," ++ max ++ "]" }, - .{ .offsets = "[0,18446744073709551616]" }, - .{ .offsets = "[0]" }, - .{ .shape = "[-4]" }, - .{ .shape = "[4294967296,4294967296]" }, - .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":[4]}}" }, - .{ .header = "{\"x\":{\"dtype\":\"U8\",\"shape\":4,\"data_offsets\":[0,4]}}" }, - .{ .header = "[]" }, - .{ .header = "{\"x\":" }, - .{ .header_len = 1 << 20 }, - .{ .header_len = std.math.maxInt(u64) }, - }; - for (cases, 0..) |case, index| { - var header_buffer: [128]u8 = undefined; - const header = case.header orelse try std.fmt.bufPrint(&header_buffer, "{{\"x\":{{\"dtype\":\"U8\",\"shape\":{s},\"data_offsets\":{s}}}}}", .{ case.shape, case.offsets }); - var name_buffer: [32]u8 = undefined; - const file = try tmp.dir.createFile(io, try std.fmt.bufPrint(&name_buffer, "{d}.safetensors", .{index}), .{ .read = true }); - defer file.close(io); - var length: [BYTES_HEADER]u8 = undefined; - std.mem.writeInt(u64, &length, case.header_len orelse header.len, .little); - try file.writePositionalAll(io, &length, 0); - try file.writePositionalAll(io, header, length.len); - try file.writePositionalAll(io, &.{ 1, 2, 3, 4 }, length.len + header.len); - - var registry: TensorRegistry = .init(allocator); - defer registry.deinit(); - const result = parseSafetensors(io, ®istry, file, ""); - if (case.valid) { - try result; - try std.testing.expectEqual(@as(u64, BYTES_HEADER + header.len), registry.tensors.get("x").?.offset); - // A name already registered, as one repeated across a checkpoint's - // shards. - try std.testing.expectError(error.DuplicateTensor, parseSafetensors(io, ®istry, file, "")); - } else { - try std.testing.expectError(error.InvalidSafetensorsHeader, result); - } - } -} - -test "addRepo adds a second checkpoint under a prefix" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - // The target is one file, the drafter an index over one shard; both name - // their tensor `w`. - const target = try tmp.dir.createDirPathOpen(io, "target", .{}); - defer target.close(io); - try testing.writeSafetensorsFile(allocator, io, target, "model.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, - }); - const drafter = try tmp.dir.createDirPathOpen(io, "drafter", .{}); - defer drafter.close(io); - try testing.writeSafetensorsFile(allocator, io, drafter, "shard.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 5, 6, 7, 8 } }, - }); - try drafter.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"shard.safetensors\"}}" }); - - var registry: TensorRegistry = try .fromRepo(allocator, io, target); - defer registry.deinit(); - try registry.addRepo(io, drafter, "drafter"); - try std.testing.expectError(error.DuplicateTensor, registry.addRepo(io, drafter, "drafter")); - - try std.testing.expectEqual(@as(usize, 2), registry.tensors.count()); - const cases = [_]struct { name: []const u8, bytes: [4]u8 }{ - .{ .name = "w", .bytes = .{ 1, 2, 3, 4 } }, - .{ .name = "drafter.w", .bytes = .{ 5, 6, 7, 8 } }, - }; - for (cases) |case| { - const tensor = registry.tensors.get(case.name).?; - try std.testing.expectEqualStrings(case.name, tensor.name); - var bytes: [4]u8 = undefined; - try tensor.read(io, &bytes); - try std.testing.expectEqualSlices(u8, &case.bytes, &bytes); - } -} - -test "fromPath opens a file, an index or a directory by a path relative to the cwd" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try testing.writeSafetensorsFile(allocator, io, tmp.dir, "layer.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{4}, .bytes = &.{ 1, 2, 3, 4 } }, - }); - for ([_][]const u8{ "layer.safetensors.index.json", "model.safetensors.index.json" }) |index| { - try tmp.dir.writeFile(io, .{ .sub_path = index, .data = "{\"weight_map\":{\"w\":\"layer.safetensors\"}}" }); - } - - for ([_][]const u8{ "layer.safetensors", "layer.safetensors.index.json", "." }) |name| { - // `tmpDir` is `.zig-cache/tmp/` under the cwd. - var path_buffer: [128]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); - var registry: TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - var bytes: [4]u8 = undefined; - try registry.tensors.get("w").?.read(io, &bytes); - try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, &bytes); - } -} - -test "fromPath refuses a file shorter than its header length or an index without a weight_map" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const files = [_]struct { []const u8, []const u8 }{ - .{ "short.safetensors", &.{ 1, 2, 3 } }, - .{ "list.safetensors.index.json", "[]" }, - .{ "unmapped.safetensors.index.json", "{\"metadata\":{\"total_size\":4}}" }, - .{ "unnamed.safetensors.index.json", "{\"weight_map\":{\"w\":4}}" }, - }; - for (files) |file| { - const name, const data = file; - try tmp.dir.writeFile(io, .{ .sub_path = name, .data = data }); - // `tmpDir` is `.zig-cache/tmp/` under the cwd. - var path_buffer: [128]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); - try std.testing.expectError(error.InvalidSafetensorsHeader, TensorRegistry.fromPath(allocator, io, path)); - } -} - -test "fromRepo reads every file an index longer than its read buffer names" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const scratch = arena.allocator(); - - // About 90 KiB of weight map, read through a 64 KiB buffer: file names - // straddle refills. - const shards = 16; - const tensors_per_shard = 64; - var index: std.Io.Writer.Allocating = .init(scratch); - try index.writer.writeAll("{\"metadata\":{\"total_size\":1024},\"weight_map\":{"); - for (0..shards) |shard| { - const file_name = try std.fmt.allocPrint(scratch, "model-{d:0>5}-of-{d:0>5}.safetensors", .{ shard + 1, shards }); - var tensors: [tensors_per_shard]testing.TensorBytes = undefined; - for (&tensors, 0..) |*tensor, expert| { - const name = try std.fmt.allocPrint(scratch, "model.layers.{d}.mlp.experts.{d}.down_proj.weight", .{ shard, expert }); - tensor.* = .{ .name = name, .dtype = "U8", .dims = &.{1}, .bytes = &.{0} }; - try index.writer.print("{s}\"{s}\":\"{s}\"", .{ if (shard + expert == 0) "" else ",", name, file_name }); - } - try testing.writeSafetensorsFile(allocator, io, tmp.dir, file_name, &tensors); - } - try index.writer.writeAll("}}"); - try std.testing.expect(index.written().len > 64 * 1024); - try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = index.written() }); - - var registry: TensorRegistry = try .fromRepo(allocator, io, tmp.dir); - defer registry.deinit(); - // Each file holds its own tensors: a misread file name would lose some. - try std.testing.expectEqual(@as(usize, shards * tensors_per_shard), registry.tensors.count()); -} - -test "checkpointFiles names the files fromPath reads" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const scratch = arena.allocator(); - - // The index names a twice and leaves out c, which sits beside its files - // as a Mistral-format `consolidated.safetensors` does. - try testing.writeSafetensorsFile(allocator, io, tmp.dir, "a.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{1} }, - .{ .name = "x", .dtype = "U8", .dims = &.{1}, .bytes = &.{2} }, - }); - try testing.writeSafetensorsFile(allocator, io, tmp.dir, "b.safetensors", &.{ - .{ .name = "y", .dtype = "U8", .dims = &.{1}, .bytes = &.{3} }, - }); - try testing.writeSafetensorsFile(allocator, io, tmp.dir, "c.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{4} }, - }); - try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"w\":\"a.safetensors\",\"y\":\"b.safetensors\",\"x\":\"a.safetensors\"}}" }); - const single = try tmp.dir.createDirPathOpen(io, "single", .{}); - defer single.close(io); - try testing.writeSafetensorsFile(allocator, io, single, "model.safetensors", &.{ - .{ .name = "w", .dtype = "U8", .dims = &.{1}, .bytes = &.{5} }, - }); - - const cases = [_]struct { []const u8, []const []const u8 }{ - .{ ".", &.{ "a.safetensors", "b.safetensors" } }, - .{ "model.safetensors.index.json", &.{ "a.safetensors", "b.safetensors" } }, - .{ "c.safetensors", &.{"c.safetensors"} }, - .{ "single", &.{"model.safetensors"} }, - }; - // `tmpDir` is `.zig-cache/tmp/` under the cwd. - var path_buffer: [128]u8 = undefined; - for (cases) |case| { - const name, const expected = case; - const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name }); - const repo, const files = try checkpointFiles(scratch, io, path); - defer repo.close(io); - try std.testing.expectEqual(expected.len, files.len); - for (expected, files) |expected_file, file| { - try std.testing.expectEqualStrings(expected_file, file); - (try repo.openFile(io, file, .{})).close(io); - } - // The registry reads these files and no other. - var registry: TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - var read: std.StringArrayHashMapUnmanaged(void) = .empty; - for (registry.tensors.values()) |tensor| try read.put(scratch, std.fs.path.basename(tensor.file_uri), {}); - try std.testing.expectEqual(read.count(), files.len); - for (files) |file| try std.testing.expect(read.contains(file)); - } - const missing = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/missing", .{&tmp.sub_path}); - try std.testing.expectError(error.FileNotFound, checkpointFiles(scratch, io, missing)); -} - pub const ModelPathResolutionError = error{ FileNotFound, - PermissionDenied, InvalidPath, - Canceled, -}; +} || std.mem.Allocator.Error; -/// Opens the model directory at `path`, relative to the cwd. A path that is -/// not a directory, such as a file's, is `InvalidPath`, logged. -pub fn resolveModelRepo( - io: std.Io, - path: []const u8, -) ModelPathResolutionError!std.Io.Dir { - return std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |err| switch (err) { - error.FileNotFound => error.FileNotFound, - error.AccessDenied, error.PermissionDenied => error.PermissionDenied, - error.Canceled => error.Canceled, - else => { - log.warn("{s} is not a model directory: {t}", .{ path, err }); - return error.InvalidPath; - }, - }; -} +pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { + switch (FileType.resolve(path)) { + .full_index, .index, .safetensors => { + // The path is a file, let's open it's folder + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; -test "resolveModelRepo refuses a file" { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); - - // `tmpDir` is `.zig-cache/tmp/` under the cwd. - var path_buffer: [128]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buffer, ".zig-cache/tmp/{s}/model.safetensors", .{&tmp.sub_path}); - try std.testing.expectError(error.InvalidPath, resolveModelRepo(io, path)); -} - -/// The entrypoint of the model directory `repo`, open, with its kind and -/// its name there: `model.safetensors.index.json`, else `model.safetensors`. -fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, EntrypointKind, []const u8 } { - // When neither opens, a denied probe is what is reported. On S3 it may - // mean absent: a bucket that may not be listed answers 403 for a - // missing key. - var denied = false; - const entrypoints = [_]struct { []const u8, EntrypointKind }{ - .{ "model.safetensors.index.json", .index }, - .{ "model.safetensors", .safetensors }, - }; - for (entrypoints) |entrypoint| { - const name, const kind = entrypoint; - if (repo.openFile(io, name, .{ .mode = .read_only })) |file| { - return .{ file, kind, name }; - } else |err| switch (err) { - error.Canceled => return ModelPathResolutionError.Canceled, - error.AccessDenied, error.PermissionDenied => denied = true, - else => {}, - } - } - return if (denied) ModelPathResolutionError.PermissionDenied else ModelPathResolutionError.FileNotFound; -} - -test "resolveModelEntrypoint falls back past a denied index and reports a denied probe" { - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const served = try tmp.dir.createDirPathOpen(io, "served", .{}); - defer served.close(io); - try served.writeFile(io, .{ .sub_path = "model.safetensors", .data = "" }); - - // The testing io, but a missing file is `PermissionDenied`: S3 answers - // 403 for a missing key to a principal without `s3:ListBucket`. - var vtable = io.vtable.*; - vtable.dirOpenFile = struct { - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - return std.testing.io.vtable.dirOpenFile(userdata, dir, sub_path, options) catch |err| switch (err) { - error.FileNotFound => error.PermissionDenied, - else => err, + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - } - }.dirOpenFile; - const unlisted: std.Io = .{ .userdata = io.userdata, .vtable = &vtable }; - const file, const kind, const name = try resolveModelEntrypoint(unlisted, served); - defer file.close(io); - try std.testing.expectEqual(.safetensors, kind); - try std.testing.expectEqualStrings("model.safetensors", name); - // `tmp.dir` holds neither entrypoint. - try std.testing.expectError(error.PermissionDenied, resolveModelEntrypoint(unlisted, tmp.dir)); - try std.testing.expectError(error.FileNotFound, resolveModelEntrypoint(io, tmp.dir)); + return dir; + }, + .unknown => { + // The path is probably a directory, let's try to open it as such + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return dir; + }, + } } -fn stringToDtype(safetensor_type: []const u8) ?DataType { - const map = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { + const default_entrypoints: []const struct { FileType, []const u8 } = &.{ + .{ .full_index, "model.safetensors.full_index.json" }, + .{ .full_index, "consolidated.safetensors.full_index.json" }, + .{ .index, "model.safetensors.index.json" }, + .{ .index, "consolidated.safetensors.index.json" }, + .{ .safetensors, "model.safetensors" }, + .{ .safetensors, "consolidated.safetensors" }, + }; - return map.get(safetensor_type); + for (default_entrypoints) |entrypoint| { + const file_type, const file_name = entrypoint; + const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; + + return .{ file_type, file }; + } + + return error.FileNotFound; +} + +const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, +}); + +fn stringToDtype(safetensor_dtype: []const u8) !DataType { + return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); + return error.UnsupportedDataType; + }; } diff --git a/zml/shape.zig b/zml/shape.zig index 7f08992e..386b594e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -117,14 +117,31 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) TagsArray { + pub fn parseTags(v: anytype) []const Shape.Tag { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_: TagsArray = .empty; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); + if (T == []const Shape.Tag) return v; + comptime validateTags(T); + return comptime frozen: { + var tagz: [v.len]Shape.Tag = undefined; + for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; + // Force the creation of a comptime constant, so that we can return a pointer to it. + const frozen_tags = tagz; + break :frozen &frozen_tags; + }; + } + + fn validateTags(T: type) void { + const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); + switch (@typeInfo(T)) { + .@"struct" => |info| { + if (!info.is_tuple) @compileError(message); + for (info.field_types) |Field| { + if (Field != @EnumLiteral()) @compileError(message); + } + if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); + }, + else => @compileError(message), } - return tags_; } /// Create a shape from a struct literal, eg: @@ -407,36 +424,6 @@ pub const Shape = struct { return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); } - /// Compares dimensions, dtype, tags, mesh, and per-axis partition specs. - pub fn eqlWithTagsAndPartitioning(self: Shape, other: Shape) bool { - if (!self.eqlWithTags(other) or self._sharding.mesh != other._sharding.mesh) return false; - for (0..self.rank()) |ax| { - if (self.partition(ax) != other.partition(ax)) return false; - } - return true; - } - - test eqlWithTagsAndPartitioning { - const other_mesh: Sharding.Mesh = .initTest(.other, .{ .batch, .feature }); - const shape = Shape.init(.{ .out = 8, .in = 16 }, .f32) - .withPartitioning(&test_mesh, .{ .out = .feature, .in = .replicated }); - const Spec = Sharding.Partitioning.Logical.Spec; - // Only the first spec is `shape`'s. - const specs: []const Spec = &.{ .init(.feature), .init(.batch), .replicated, .open }; - for (specs, 0..) |spec, i| { - const other = shape.withPartitioning(&test_mesh, .{ .out = spec, .in = .replicated }); - try testing.expect(shape.eqlWithTags(other)); - try testing.expectEqual(i == 0, shape.eqlWithTagsAndPartitioning(other)); - } - try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withPartitioning(&other_mesh, .{ .out = .feature, .in = .replicated }))); - try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.withDtype(.bf16))); - try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.setDim(.out, 9))); - try testing.expect(!shape.eqlWithTagsAndPartitioning(shape.rename(.{ .out = .features }))); - try testing.expect(!shape.eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8 }, .f32))); - try testing.expect(!shape.withDefaultPartitioning().eqlWithTagsAndPartitioning(Shape.init(.{ .out = 8, .in = 16 }, .f32))); - try testing.expect(Shape.init(.{}, .f32).eqlWithTagsAndPartitioning(Shape.init(.{}, .f32))); - } - /// Format the shape. /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { @@ -838,6 +825,18 @@ pub const Shape = struct { } } + pub fn assertTags(self: Shape, tagz: anytype) void { + const T = @TypeOf(tagz); + if (T == Shape) { + return self.assertTags(tagz.tags()); + } else if (T != []const Shape.Tag) { + return self.assertTags(comptime parseTags(tagz)); + } + + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); + stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); + } + pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; @@ -857,11 +856,12 @@ pub const Shape = struct { .grid = undefined, }; - /// Partitions this shape on `mesh` as `Tensor.withPartitioning` takes `partitioning`, except - /// that axes it leaves unspecified are `.replicated`. + /// Returns shape metadata with the requested sharding on `mesh`. + /// Omitted dimensions in a mapping are replicated, including during compilation. + /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { - return self.withPartitioning(mesh, Sharding.Partitioning.Logical.init(partitioning).resolve(self.tags(), mesh, .replicated)); + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); } var res = self; res._sharding = .{ .mesh = mesh, .partition = partitioning }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 5dd62ddc..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -75,14 +75,11 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. - /// Note: sharding information is discarded, the output tensor is considered open for replication. - /// Use _resultPropagateSharding when the shape sharding should be propagated. + /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, + /// and open sharding for the partitioner to infer from graph constraints. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; - res._shape._sharding.partition = .open(sh.rank()); + res._shape._sharding = .open; if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. @@ -95,16 +92,16 @@ pub const Tensor = struct { return res; } + /// Internal: creates a constructor result and honors `sh`'s partitioning when + /// it specifies a mesh. pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { - var res = _result(sh, val); - res._shape._sharding = sh._sharding; - return res; + const result = _result(sh, val); + const mesh = sh._sharding.mesh orelse return result; + return result.withPartitioning(mesh, sh._sharding.partition); } - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. - /// mesh is `unknown` + /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. + /// Dimension tags are unknown and sharding is open for partitioner inference. pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -116,7 +113,7 @@ pub const Tensor = struct { ._dtype = ctx.dtype(ranked_tensor.elementType()), ._dims = .empty, ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, - ._sharding = .replicated, + ._sharding = .open, }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); @@ -166,14 +163,22 @@ pub const Tensor = struct { return res; } - /// Specify the mesh of the input tensor. - /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get - /// a known mesh from the compilation options. - /// * partitioning: a `Partitioning`, or anything `Partitioning.Logical.init` accepts: a struct - /// mapping this tensor's axis tags to logical mesh axes, `.replicated` or `.open`. - /// Axes it leaves unspecified are `.open`. + /// Declares this tensor's sharding on `mesh_`. + /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. + /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension + /// tags to logical mesh axes, `.replicated`, or `.open`. /// - /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + /// During compilation, emits a graph constraint. For example, + /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` + /// and leaves omitted dimensions open for inference. Use `.replicated` to + /// keep a dimension whole on each shard. + /// + /// Outside compilation, updates the tensor's shape to describe an executable + /// input's sharding. Omitted dimensions are replicated and no graph constraint is + /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. + /// + /// Executable inputs and explicitly annotated outputs must have no open + /// dimensions. An output with no declared sharding defaults to replication. pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { if (@TypeOf(mesh_) == @EnumLiteral()) { const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); @@ -181,7 +186,8 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const parsed = Sharding.Partitioning.Logical.init(partitioning).resolve(self._shape.tags(), mesh_, .open); + const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); return self.withPartitioning(mesh_, parsed); } @@ -224,11 +230,14 @@ pub const Tensor = struct { }, }; - return _resultPropagateSharding(partitioned_shape, op_result); + var res = _result(partitioned_shape, op_result); + res._shape = partitioned_shape; + return res; } - /// Force the input tensor to be fully replicated. - /// This can be detrimental for performance. + /// Emits a graph constraint requiring a complete copy on every device. + /// Requires active compilation. For a replicated input or buffer description, + /// use the default sharding from `Tensor.init` or `Shape.init`. pub fn replicate(self: Tensor) Tensor { const compiler = Compiler.current(); const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); @@ -269,11 +278,17 @@ pub const Tensor = struct { try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } - /// Copy the given tensor to the specified memory. + /// Copy the given tensor to the specified memory, preserving its sharding. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } @@ -300,7 +315,8 @@ pub const Tensor = struct { ctx.location, ).appendTo(currentBlock()); - const res = _result(self._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._shape._sharding = self._shape._sharding; ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } @@ -847,7 +863,7 @@ pub const Tensor = struct { fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -888,7 +904,8 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; + const result = floats.convert(shape_.dtype()); + return .{ rng, _resultPropagateSharding(shape_, result.value()) }; } test uniform { @@ -1517,7 +1534,7 @@ pub const Tensor = struct { pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { const T = @TypeOf(tags); if (comptime T == Shape.Tag or T == @EnumLiteral()) { - return lhs.dotAxes(rhs, .{tags}); + return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); } const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); @@ -2183,15 +2200,14 @@ pub const Tensor = struct { pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; - // `len` is checked before `end`, whose default would otherwise take the rest of the axis. const end = if (s.singleton) start + 1 - else if (s.len != to_the_end) - start + s.len else if (s.end == to_the_end) d else if (s.end < 0) d + s.end + else if (s.len != to_the_end) + start + s.len else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; @@ -2342,7 +2358,6 @@ pub const Tensor = struct { .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, - .{ 1, Tensor.Slice{ .start = 1, .len = 2 }, [4]f32{ 1, 2, 6, 7 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; var exe = try zml.module.compile( @@ -2709,7 +2724,7 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), res_shape), currentLoc(), ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); + return _resultPropagateSharding(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -2784,7 +2799,8 @@ pub const Tensor = struct { } pub fn zeroes(sh: Shape) Tensor { - return Tensor.constant(sh.dtype().zero()).broad(sh); + const result = Tensor.constant(sh.dtype().zero()).broad(sh); + return _resultPropagateSharding(sh, result.value()); } pub fn uninitialized(sh: Shape) Tensor { @@ -2799,7 +2815,7 @@ pub const Tensor = struct { .appendTo(currentBlock()); const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); - return _result(sh, tensor.result(0)); + return _resultPropagateSharding(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. @@ -2807,7 +2823,7 @@ pub const Tensor = struct { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); - return _result(sh, constant_op.result(0)); + return _resultPropagateSharding(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2973,7 +2989,7 @@ pub const Tensor = struct { var res_shape = self._shape; const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + res_shape._tags.insertSlice(ax, tags_) catch unreachable; return self.reshape(res_shape); } @@ -4810,7 +4826,7 @@ pub const Tensor = struct { fn body(body_ctx: @This(), _: void) void { ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); } - }).body, .{ .input = input, .name = full_name }, {}); + }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); }, .oneapi, .neuron, .furiosa => {}, } diff --git a/zml/testing.zig b/zml/testing.zig index d1e6faf7..5e3d6333 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -57,8 +57,8 @@ fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) ! } else .{ slice_or_buffer, false }; } -/// Testing utility. Accepts both zml.Buffer and zml.Slice; a zml.Buffer is copied to the host for -/// comparison. +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the +/// host for comparison ! pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; var left: Slice, const should_free_left = try toSlice(allocator, io, left_); @@ -365,189 +365,20 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } -/// `device_count` devices for a loader test: the accelerator the tests run -/// on (`env`, shared and never freed) when it has that many, otherwise a CPU -/// platform with exactly that many, freed by `deinit`. `physical` lays the -/// first `device_count` devices on one axis, so a mesh over it (`mesh`) -/// places on exactly those, whatever the platform has. Must not move after -/// `init`: meshes point at `physical`. -pub const Devices = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - owned: ?*Platform, - physical: Sharding.PhysicalMesh, - - pub fn init(self: *Devices, allocator: std.mem.Allocator, io: std.Io, device_count: usize) !void { - const shared = env(); - const owned: ?*Platform = if (shared.target != .cpu and shared.devices.len >= device_count) - null - else - try Platform.init(allocator, io, .cpu, .{ .cpu = .{ .device_count = @intCast(device_count) } }); - errdefer if (owned) |platform| platform.deinit(allocator, io); - const platform: *const Platform = if (owned) |p| p else shared; - const nodes = try allocator.alloc(Sharding.PhysicalNode, device_count); - defer allocator.free(nodes); - for (nodes, platform.devices[0..device_count]) |*node, device| node.* = .device(device); - self.* = .{ - .allocator = allocator, - .io = io, - .platform = platform, - .owned = owned, - .physical = try .fromTree(allocator, platform.target, .axis(.bus, .tree, nodes)), - }; - } - - pub fn deinit(self: *Devices) void { - self.physical.deinit(self.allocator); - if (self.owned) |platform| platform.deinit(self.allocator, self.io); - self.* = undefined; - } - - /// A mesh named `name` whose one axis, `model`, spans the devices. - pub fn mesh(self: *const Devices, name: []const u8) !Sharding.Mesh { - return .init(name, &self.physical, .mesh(.{ .model = .high_bandwidth }), .parseBindings(.{ .model = .bus })); - } -}; - -pub const TensorBytes = struct { - name: []const u8, - /// The safetensors dtype name, such as "U16". - dtype: []const u8, - dims: []const i64, - bytes: []const u8, -}; - -/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order, -/// and registers them from it. -pub fn writeSafetensors( - allocator: std.mem.Allocator, - io: std.Io, - dir: std.Io.Dir, - sub_path: []const u8, - registry: *zml.safetensors.TensorRegistry, - tensors: []const TensorBytes, -) !void { - try writeSafetensorsFile(allocator, io, dir, sub_path, tensors); - const file = try dir.openFile(io, sub_path, .{}); - defer file.close(io); - try zml.safetensors.parseSafetensors(io, registry, file, ""); +fn BufferizedWithArgs(comptime T: type) type { + return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } -/// Writes `tensors` as the safetensors file `sub_path` of `dir`, in order. -pub fn writeSafetensorsFile( - allocator: std.mem.Allocator, - io: std.Io, - dir: std.Io.Dir, - sub_path: []const u8, - tensors: []const TensorBytes, -) !void { - var header: std.Io.Writer.Allocating = .init(allocator); - defer header.deinit(); - const writer = &header.writer; - try writer.writeByte('{'); - var offset: usize = 0; - for (tensors, 0..) |tensor, index| { - if (index != 0) try writer.writeByte(','); - try writer.print("\"{s}\":{{\"dtype\":\"{s}\",\"shape\":[", .{ tensor.name, tensor.dtype }); - for (tensor.dims, 0..) |dim, axis| try writer.print("{s}{d}", .{ if (axis != 0) "," else "", dim }); - try writer.print("],\"data_offsets\":[{d},{d}]}}", .{ offset, offset + tensor.bytes.len }); - offset += tensor.bytes.len; - } - try writer.writeByte('}'); - - const file = try dir.createFile(io, sub_path, .{}); - defer file.close(io); - var header_len: [8]u8 = undefined; - std.mem.writeInt(u64, &header_len, header.written().len, .little); - try file.writePositionalAll(io, &header_len, 0); - var position: u64 = header_len.len; - try file.writePositionalAll(io, header.written(), position); - position += header.written().len; - for (tensors) |tensor| { - try file.writePositionalAll(io, tensor.bytes, position); - position += tensor.bytes.len; - } -} - -/// Distinct, nonzero bytes for test tensor number `seed`. -pub fn pattern(bytes: []u8, seed: u8) void { - for (bytes, 0..) |*byte, index| byte.* = (@as(u8, @truncate(index *% 29)) +% seed *% 53) | 1; -} - -/// A store over a copy of `registry`, for a test that builds several stores -/// over one checkpoint: a store takes its registry's entries. -pub fn storeOf(allocator: std.mem.Allocator, registry: *const zml.safetensors.TensorRegistry, opts: zml.io.TensorStore.Options) !zml.io.TensorStore { - var copy: zml.safetensors.TensorRegistry = .init(allocator); - defer copy.deinit(); - for (registry.tensors.values()) |tensor| try copy.registerTensor(tensor); - return .init(allocator, ©, opts); -} - -/// Loads `model` from `store` with a loader of its own and returns its -/// buffers, which the caller frees with `mem.deinitBufferized`. A recipe -/// for tests: production code creates its loader before building the -/// model, so the loader's set-up runs meanwhile. -pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - model: anytype, - store: *const zml.io.TensorStore, -) !zml.Bufferized(@TypeOf(model.*)) { - var buffers = try zml.mem.bufferize(allocator, model); - errdefer zml.mem.deinitBufferized(allocator, &buffers); - var loader = try zml.io.Loader.init(allocator, io, platform, .{}); - defer loader.deinit(); - try loader.start(model, &buffers, store, .none); - try loader.wait(); - return buffers; -} - -/// Reads every shard of `buffer` back, in its mesh's device order, and -/// checks that it lives in `memory`'s space and holds that device's box of -/// `global`, the whole tensor's row-major bytes (sub-byte elements packed). -/// A buffer without a mesh is replicated on the platform's devices. On -/// CPU/neuron/metal every memory kind is the one memory space, so only -/// CUDA/ROCm/oneAPI tell them apart. -pub fn expectShards(allocator: std.mem.Allocator, io: std.Io, buffer: *const zml.Buffer, global: []const u8, memory: zml.io.TensorStore.Memory) !void { - const platform = buffer._platform; - const shape = platform.placedShape(buffer.shape()).packedShape(); - try std.testing.expectEqual(shape.byteSize(), global.len); - const placement: Sharding.Placement = try .init(&shape); - const devices = placement.mesh().?.devicesInCanonicalOrder(); - try std.testing.expectEqual(devices.len, buffer._shards.len); - const shard_bytes = placement.localShape(shape).byteSize(); - const actual = try allocator.alloc(u8, shard_bytes); - defer allocator.free(actual); - const expected = try allocator.alloc(u8, shard_bytes); - defer allocator.free(expected); - const rank = shape.rank(); - const strides = shape.computeByteStrides(); - var shards = buffer.shards(); - for (devices, 0..) |device, device_index| { - const shard = shards.next().?; - try std.testing.expect(shard._pjrt_buffer.memory(platform.pjrt_api) == platform.devices[device.id].memory(memory.space()).?.pjrt_memory); - try shard.toHost(io, actual); - // The device's box, one run of the last axis per index of the others. - const box = placement.slices(device_index, shape); - const run: usize = if (rank == 0) shape.dtype().sizeOf() else @intCast(box.get(rank - 1).size * strides.get(rank - 1)); - var index: [zml.Shape.MAX_RANK]i64 = @splat(0); - var cursor: usize = 0; - while (cursor < expected.len) : (cursor += run) { - var offset: i64 = 0; - for (0..rank) |axis| offset += (box.get(axis).start + index[axis]) * strides.get(axis); - @memcpy(expected[cursor..][0..run], global[@intCast(offset)..][0..run]); - var axis = rank -| 1; - while (axis > 0) { - axis -= 1; - index[axis] += 1; - if (index[axis] < box.get(axis).size) break; - index[axis] = 0; - } +fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { + var count: usize = 0; + var it = store.store.registry.iterator(); + while (it.next()) |entry| { + if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { + count += 1; } - try std.testing.expectEqualSlices(u8, expected, actual); } + + return count; } /// Test a layer implementation against reference activations stored in a TensorStore. @@ -588,12 +419,12 @@ pub fn testLayer( const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const store_input = activation_store.withPrefix(input_name); - const expected_input_count = store_input.count(); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -611,7 +442,7 @@ pub fn testLayer( const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; const store_output = activation_store.withPrefix(output_name); - const expected_output_count = store_output.count(); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -622,27 +453,34 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try load(allocator, io, platform, &args, activation_store.store); - defer zml.mem.deinitBufferized(allocator, &args_buffers); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); + defer zml.meta.visit(struct { + fn cb(_: void, b: *zml.Buffer) void { + b.deinit(); + } + }.cb, {}, &args_buffers); exe_args.set(.{ layer_weights, args_buffers }); var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); - try exe.call(io, exe_args, &exe_results, .{ .wait = true }); + exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); defer for (results) |*result| result.deinit(); var failed: bool = false; + var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); - const tensor = store_output.get(subkey) orelse return error.TensorNotFound; - const expected_slice: zml.Slice = try .alloc(allocator, tensor.shape); + var reader = try store_output.getReader(subkey, io, &reader_buffer); + defer reader.deinit(); + const shape = store_output.getShape(subkey).?; + const expected_slice: zml.Slice = try .alloc(allocator, shape); errdefer expected_slice.free(allocator); - try tensor.read(io, expected_slice.data()); + try reader.interface.readSliceAll(expected_slice.data()); break :s expected_slice; }; defer expected_slice.free(allocator); @@ -670,39 +508,6 @@ pub fn testLayer( } } -test testLayer { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = env(); - - const Scale = struct { - weight: zml.Tensor, - - pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { - return x.mul(self.weight); - } - }; - const weight = [_]f32{ 2, -1, 0.5, 3 }; - const input = [_]f32{ 1, 2, 3, 4 }; - const output = [_]f32{ 2, -2, 1.5, 12 }; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: zml.safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try writeSafetensors(allocator, io, tmp.dir, "layer.safetensors", ®istry, &.{ - .{ .name = "scale.weight", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&weight) }, - .{ .name = "x.in.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&input) }, - .{ .name = "x.out.0", .dtype = "F32", .dims = &.{4}, .bytes = std.mem.sliceAsBytes(&output) }, - }); - var store: zml.io.TensorStore = .init(allocator, ®istry, .{}); - defer store.deinit(); - - const layer: Scale = .{ .weight = store.view().withPrefix("scale").createReplicatedTensor("weight", null) }; - var layer_weights = try load(allocator, io, platform, &layer, &store); - defer zml.mem.deinitBufferized(allocator, &layer_weights); - try testLayer(allocator, io, platform, layer, .forward, store.view(), "x", layer_weights, &.{}, .{}); -} - fn center(slice: anytype, i: usize) @TypeOf(slice) { const c = 20; const start = if (i < c) 0 else i - c; @@ -713,27 +518,3 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { return std.testing.expectEqual(expected, actual); } - -test load { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var registry: zml.safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - var bytes: [8]u8 = undefined; - pattern(&bytes, 1); - try writeSafetensors(allocator, io, tmp.dir, "weights.safetensors", ®istry, &.{ - .{ .name = "weight", .dtype = "U8", .dims = &.{8}, .bytes = &bytes }, - }); - var store = try storeOf(allocator, ®istry, .{}); - defer store.deinit(); - // The store took a copy: the registry still holds its entries. - try std.testing.expectEqual(1, registry.tensors.count()); - - const Model = struct { weight: zml.Tensor }; - const model: Model = .{ .weight = store.view().createReplicatedTensor("weight", null) }; - var buffers = try load(allocator, io, env(), &model, &store); - defer zml.mem.deinitBufferized(allocator, &buffers); - try expectShards(allocator, io, &buffers.weight, &bytes, .default); -} diff --git a/zml/zml.zig b/zml/zml.zig index 23d12e75..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel.zig"); +pub const kernel = @import("kernel/kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; @@ -81,3 +81,9 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +}