From fa8008744469e916c7ee485b665fe4527d8310ce Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sat, 7 Nov 2026 22:30:22 +0000 Subject: [PATCH 1/6] zml: break direct cycle Between `zml/attention/attention.zig` and `zml/kernels/flash_attention/flash_attention.zig` --- zml/BUILD.bazel | 2 + zml/attention/Options.zig | 3 + zml/attention/attention.zig | 6 +- zml/kernels/flash_attention/README.md | 2 +- .../flash_attention/flash_attention.zig | 330 +----------------- zml/kernels/flash_attention/tests.zig | 330 ++++++++++++++++++ 6 files changed, 340 insertions(+), 333 deletions(-) create mode 100644 zml/attention/Options.zig create mode 100644 zml/kernels/flash_attention/tests.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index dbb2ad33..4d58fc77 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,6 +78,7 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", + "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -88,6 +89,7 @@ zig_library( "io.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", "kernels/paged_attention/paged_attention.zig", diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 039c6628..0ac2a95c 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,11 +93,7 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = struct { - /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every - /// key, as in an encoder or a diffusion transformer, and token_index is ignored. - causal: bool = true, -}; +pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index b5fe7682..81a35cf4 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [flash_attention.zig](flash_attention.zig) compare results +The tests in [tests.zig](tests.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 78602bd0..3fb898db 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,9 +6,11 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -126,7 +128,7 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), @@ -135,329 +137,3 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: Metadata = .init(.fromBackend(backend)); - const parameters: Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..d93bd373 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,330 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} From dc71ba0e34794064a30bab12536a6b50b708c927 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sun, 8 Nov 2026 13:59:06 +0000 Subject: [PATCH 2/6] zml: break direct cycle in paged_attention Redeclare `AttentionOptions` and `AttentionMask` in backend implementation and add convenience function to convert. Another solution would have been to make `AttentionOptions` and `AttentionMask` their own module. --- zml/kernels/paged_attention/cute/cute.zig | 33 ++++++++++++- .../paged_attention/cute/sm10x/sm10x.zig | 3 +- .../paged_attention/cute/sm120/sm120.zig | 3 +- zml/kernels/paged_attention/metal/metal.zig | 31 +++++++++++- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 31 +++++++++++- .../paged_attention/paged_attention.zig | 43 +++++++++------- .../paged_attention/stablehlo/stablehlo.zig | 49 ++++++++++++++++--- zml/kernels/paged_attention/tcl/tcl.zig | 45 ++++++++++++++--- zml/kernels/paged_attention/triton/triton.zig | 31 +++++++++++- 9 files changed, 225 insertions(+), 44 deletions(-) diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ea4f36ab..8149b8ff 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,7 +1,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const paged = @import("../paged_attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); const kernel = @import("attention.zig"); @@ -64,7 +63,37 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index af068bad..afc4298e 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,7 +2,6 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -19,7 +18,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 292e4a9f..6645abf1 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,7 +1,6 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,7 +17,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e2d68398..94880001 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -4,7 +4,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); @@ -58,6 +57,36 @@ pub const Parameters = struct { } }; +/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 42e1861f..0fbb40e4 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,6 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -218,6 +217,36 @@ pub const mosaic_tpu = struct { }; } + /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } + }; + + /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, + }; + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); const prepared = prepareInputs(parameters, q, kv_cache); diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 81f7ec3e..b17b1df1 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -178,12 +178,27 @@ pub const AttentionMask = union(enum) { .indices => null, }; } + + fn toBackendStruct(self: AttentionMask, comptime T: type) T { + return switch (self) { + .standard => |v| .{ .standard = .{ .causal = v.causal, .sliding_window = v.sliding_window } }, + .indices => |v| .{ .indices = v }, + }; + } }; pub const AttentionOptions = struct { mask: AttentionMask = .{ .standard = .{} }, scale: ?f32 = null, sink: ?zml.Tensor = null, + + fn toBackendStruct(self: AttentionOptions, comptime T: type) T { + return .{ + .mask = self.mask.toBackendStruct(@FieldType(T, "mask")), + .scale = self.scale, + .sink = self.sink, + }; + } }; pub const KvCache = union(enum) { @@ -232,19 +247,6 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } - - pub fn getPage( - self: KvCache, - page_index: zml.Tensor, - ) [2]zml.Tensor { - return switch (self) { - .split => |split| .{ - split.k.slice(split.k.axis(.page), .dynSingle(page_index)), - split.v.slice(split.k.axis(.page), .dynSingle(page_index)), - }, - .dense => @panic("TODO"), - }; - } }; pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); @@ -265,24 +267,27 @@ const Attention = struct { fn forward(args: @This(), _: zml.Shape) zml.Tensor { return switch (args.parameters) { .cute => |p| switch (args.kv_cache) { - .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts.toBackendStruct(cute.AttentionOptions)), .dense => @panic("CuTe paged attention requires split KV pages"), }, .triton => |triton_parameters| switch (args.kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts.toBackendStruct(triton.paged.AttentionOptions)), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts.toBackendStruct(tpu.mosaic_tpu.AttentionOptions)), }, .metal => |metal_parameters| switch (args.kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts.toBackendStruct(metal.AttentionOptions)), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts.toBackendStruct(stablehlo.AttentionOptions)), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, - .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), .tcl => |params| switch (args.kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts.toBackendStruct(tcl.paged.AttentionOptions)), .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), }, }; diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 7ee4df03..96555ab0 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,41 @@ const zml = @import("../../../zml.zig"); -const KvCache = @import("../paged_attention.zig").KvCache; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); +/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,12 +43,12 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = kv_cache.split.k.dim(.k_chunk); + const page_size = k.dim(.k_chunk); const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -38,7 +67,8 @@ pub fn pagedAttention( const AttentionLoop = struct { q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -66,7 +96,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -134,7 +164,10 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index c9993af9..195cff86 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -9,7 +9,6 @@ const builder = @import("dsl/tcl/builder"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const paged_attention = @import("../paged_attention.zig"); const triton = @import("../triton/triton.zig"); /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to @@ -47,9 +46,39 @@ pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; + /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } + }; + + /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, + }; + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -76,14 +105,14 @@ pub const paged = struct { q: Tensor, k_cache: Tensor, v_cache: Tensor, - opts: paged_attention.AttentionOptions, + opts: AttentionOptions, fn body(self: Shard, _: zml.Shape) Tensor { return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); } }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to @@ -1664,11 +1693,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged.AttentionOptions) Tensor { return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: paged_attention.AttentionOptions = .{ + const opts: paged.AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1742,11 +1771,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ + var opts_d: zml.Bufferized(paged.AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(paged.AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 67416285..e70fba01 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,7 +4,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); @@ -269,6 +268,36 @@ pub const paged = struct { } }; + /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionMask = union(enum) { + standard: struct { + causal: bool = true, + sliding_window: ?u32 = null, + }, + indices: zml.Tensor, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } + }; + + /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` + pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, + }; + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); From 224f65d0464631acd54dc94d35ad7eeb2d069668 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sun, 8 Nov 2026 14:16:46 +0000 Subject: [PATCH 3/6] changed my mind Declare AttentionOptions in another file that will become a module. Argument is that with the multiplication of backend it's going to be a pain to have many copies of the same struct. --- zml/BUILD.bazel | 1 + zml/kernels/paged_attention/cute/cute.zig | 33 +--------- zml/kernels/paged_attention/cute/fixtures.zig | 2 + zml/kernels/paged_attention/metal/metal.zig | 32 +--------- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 31 +-------- zml/kernels/paged_attention/options.zig | 36 +++++++++++ .../paged_attention/paged_attention.zig | 63 +++---------------- .../paged_attention/stablehlo/stablehlo.zig | 31 +-------- zml/kernels/paged_attention/tcl/tcl.zig | 41 ++---------- zml/kernels/paged_attention/triton/triton.zig | 33 +--------- 10 files changed, 59 insertions(+), 244 deletions(-) create mode 100644 zml/kernels/paged_attention/options.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4d58fc77..e4ce12eb 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -92,6 +92,7 @@ zig_library( "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", + "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 8149b8ff..ef78cc49 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,9 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); -const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -63,36 +64,6 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index aafb8bf1..c0815d1d 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,5 +1,7 @@ const std = @import("std"); + const zml = @import("zml"); + const cases = @import("cases.zig"); pub const Fixture = struct { diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index 94880001..e8144e3b 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,7 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); - +const AttentionOptions = @import("../options.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); @@ -57,36 +57,6 @@ pub const Parameters = struct { } }; -/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 0fbb40e4..84716379 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,6 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -217,36 +218,6 @@ pub const mosaic_tpu = struct { }; } - /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } - }; - - /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, - }; - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); const prepared = prepareInputs(parameters, q, kv_cache); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index b17b1df1..92e77888 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); @@ -151,56 +152,6 @@ pub const Parameters = union(Backend) { } }; -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: zml.Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } - - fn toBackendStruct(self: AttentionMask, comptime T: type) T { - return switch (self) { - .standard => |v| .{ .standard = .{ .causal = v.causal, .sliding_window = v.sliding_window } }, - .indices => |v| .{ .indices = v }, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, - - fn toBackendStruct(self: AttentionOptions, comptime T: type) T { - return .{ - .mask = self.mask.toBackendStruct(@FieldType(T, "mask")), - .scale = self.scale, - .sink = self.sink, - }; - } -}; - pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -267,27 +218,27 @@ const Attention = struct { fn forward(args: @This(), _: zml.Shape) zml.Tensor { return switch (args.parameters) { .cute => |p| switch (args.kv_cache) { - .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts.toBackendStruct(cute.AttentionOptions)), + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), .dense => @panic("CuTe paged attention requires split KV pages"), }, .triton => |triton_parameters| switch (args.kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts.toBackendStruct(triton.paged.AttentionOptions)), + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts.toBackendStruct(tpu.mosaic_tpu.AttentionOptions)), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), }, .metal => |metal_parameters| switch (args.kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts.toBackendStruct(metal.AttentionOptions)), + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .stablehlo => |params| switch (args.kv_cache) { - .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts.toBackendStruct(stablehlo.AttentionOptions)), + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .tcl => |params| switch (args.kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts.toBackendStruct(tcl.paged.AttentionOptions)), + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), }, }; diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 96555ab0..55ee01d5 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,36 +1,7 @@ const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); -/// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -/// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 195cff86..881593aa 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -3,12 +3,13 @@ //! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); -const stdx = @import("stdx"); const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to @@ -46,36 +47,6 @@ pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; - /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } - }; - - /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, - }; - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { @@ -1693,11 +1664,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged.AttentionOptions) Tensor { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: paged.AttentionOptions = .{ + const opts: AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1771,11 +1742,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(paged.AttentionOptions) = .{ + var opts_d: zml.Bufferized(AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(paged.AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index e70fba01..2a12e012 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,9 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; +const AttentionOptions = @import("../options.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -268,36 +269,6 @@ pub const paged = struct { } }; - /// See AttentionMask in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionMask = union(enum) { - standard: struct { - causal: bool = true, - sliding_window: ?u32 = null, - }, - indices: zml.Tensor, - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } - }; - - /// See AttentionOptions in `zml/kernels/paged_attention/paged_attention.zig` - pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, - }; - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); From fe752fcba2935ec50f32c1b25f8b710f79137ef8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sun, 8 Nov 2026 14:53:47 +0000 Subject: [PATCH 4/6] zml/kernels/moe: break cycle `zml/kernels/moe.zig` is not imported by backend implementation anymore. I introduced `options.zig` to contain the shared options. I also introduced `weights.zig` to expose the unpacking helpers. --- zml/BUILD.bazel | 2 + zml/kernels/moe/cute/cute.zig | 6 +- zml/kernels/moe/cute/sm10x/moe.zig | 3 +- .../cutlass_flashinfer/cutlass_flashinfer.zig | 27 ++-- zml/kernels/moe/fly/fly.zig | 12 +- zml/kernels/moe/metal/metal.zig | 14 ++- zml/kernels/moe/moe.zig | 119 +----------------- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 8 +- zml/kernels/moe/options.zig | 117 +++++++++++++++++ zml/kernels/moe/stablehlo/stablehlo.zig | 12 +- zml/kernels/moe/tcl/tcl.zig | 4 +- zml/kernels/moe/tests.zig | 19 +-- zml/kernels/moe/triton/moe.zig | 5 +- zml/kernels/moe/triton/triton.zig | 25 ++-- zml/kernels/moe/weights.zig | 10 ++ 15 files changed, 212 insertions(+), 171 deletions(-) create mode 100644 zml/kernels/moe/options.zig create mode 100644 zml/kernels/moe/weights.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e4ce12eb..3c0ace0b 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,7 +91,9 @@ zig_library( "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", + "kernels/moe/options.zig", "kernels/moe/tests.zig", + "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 708ea757..14f1b090 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,6 +3,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +const Options = @import("../options.zig").Options; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -24,7 +26,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -41,7 +43,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index 479e6516..a2eb1792 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -12,6 +12,7 @@ pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { @@ -252,7 +253,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index e2c31c44..118593a8 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -1,9 +1,14 @@ const std = @import("std"); -const stdx = @import("stdx"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); +const stdx = @import("stdx"); + const zml = @import("../../../zml.zig"); +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); @@ -13,14 +18,14 @@ pub const Options = struct { /// Device used at graph-construction time to select the architecture /// library and size the XLA-owned scratch buffer. workspace_query_device: i32, - activation: zml.moe.Activation, + activation: Activation, enable_pdl: bool, gemm1_tactic: i32, /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to /// obtain the first valid GEMM2 index. gemm2_tactic: i32, - pub fn fromOptions(opts: zml.moe.Options) Options { + pub fn fromGlobalOptions(opts: GlobalOptions) Options { return .{ .workspace_query_device = 0, .activation = opts.activation, @@ -31,7 +36,7 @@ pub const Options = struct { } }; -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); @@ -79,8 +84,6 @@ const DeviceRunner = struct { runner: *fi_cutlass_moe.Runner, }; -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; - pub const Runners = struct { runners: [max_num_devices]?DeviceRunner = @splat(null), @@ -151,7 +154,7 @@ fn currentRunners() !*Runners { return runners orelse error.RunnersNotLoaded; } -fn cutlassActivation(activation: zml.moe.Activation) i32 { +fn cutlassActivation(activation: Activation) i32 { return switch (activation) { .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, @@ -309,7 +312,7 @@ pub fn fc1BlockScaleShape( num_experts: i64, hidden_size: i64, intermediate_size: i64, - activation: zml.moe.Activation, + activation: Activation, ) zml.Shape { const rows = switch (activation) { .silu, .gelu, .relu => intermediate_size, @@ -505,7 +508,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { if (comptime !platforms.isEnabled(.cuda)) { @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); @@ -519,13 +522,13 @@ pub fn fusedExperts( // The runner ignores routes whose local expert id is -1. const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - const runner_options: Options = .fromOptions(opts); + const runner_options: Options = .fromGlobalOptions(opts); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index 59d71dc9..e30292cf 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,8 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; const fly = @import("moe.zig"); -const ProjectionLayout = zml.moe.ProjectionLayout; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -20,7 +24,7 @@ pub fn fusedExperts( routing_: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; @@ -103,7 +107,7 @@ pub fn fusedExperts( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -131,7 +135,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: zml.moe.Options) bool { +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 9d397a46..988a4f51 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,12 +1,16 @@ const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -67,7 +71,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); @@ -79,8 +83,8 @@ pub fn fusedExperts( const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up = zml.moe.unpackedWeight(gate_up_); - const down = zml.moe.unpackedWeight(down_); + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { @@ -138,7 +142,7 @@ pub fn fusedExperts( zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, ), gate_up_global_scale, expert_ids); - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 7ad85c20..b3ca9814 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -9,6 +9,8 @@ pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zi pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -20,60 +22,6 @@ test { _ = @import("tests.zig"); } -pub const ProjectionLayout = enum { concatenated, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - }; -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, -}; - pub const Backend = enum { stablehlo, cute, @@ -210,14 +158,6 @@ pub const LocalRouting = struct { } }; -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -375,58 +315,3 @@ pub fn forwardMoe( .out_specs = partitioning.out_specs, }); } - -pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} - -pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { - const x = input.convert(.f32); - return switch (activation) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - }; - - break :b switch (activation) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; -} diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index c583b3e4..4b33b585 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,6 +5,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -12,7 +14,7 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -215,7 +217,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); const expert_map_ = routing.expertMap(); @@ -276,7 +278,7 @@ pub fn fusedExperts( gate_up_out = gate_up_out.add(bias_per_token); } - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/options.zig new file mode 100644 index 00000000..aa68b06a --- /dev/null +++ b/zml/kernels/moe/options.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { concatenated, interleaved }; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Whether the two projections occupy contiguous halves or alternate. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 25c69c5c..7f84835b 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,13 +1,17 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { +pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -148,13 +152,13 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: zml.moe.Activation, + activation: Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, }; /// Negative expert IDs denote nonlocal routes and contribute zero, including bias. @@ -187,7 +191,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 991280fd..698c4d30 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -12,17 +12,17 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; +const Options = @import("../options.zig").Options; /// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.ids; const topk_weights = routing.weights; stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index aa0be86f..0abb3462 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,12 +5,15 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Placement = @import("moe.zig").RoutingWeightPlacement; +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const GlobalOptions = @import("options.zig").Options; +const Activation = @import("options.zig").Activation; +const Placement = @import("options.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, - activation: zml.moe.Activation = .{ .swiglu = .{} }, + activation: Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -197,7 +200,7 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, .interleaved => interleave(tensor), @@ -271,7 +274,7 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: zml.moe.Options = .{ + const options: GlobalOptions = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -305,7 +308,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]zml.moe.Activation{ +const gated_activations = [_]Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -315,8 +318,8 @@ const gated_activations = [_]zml.moe.Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]zml.moe.Activation{ +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -354,7 +357,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const zml.moe.Activation, + activations: []const Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 3eb457da..0776a590 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,6 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1026,7 +1027,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1092,7 +1093,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 348f5a90..bae76744 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,10 +6,13 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const kernels = @import("moe.zig"); const toDType = zml.kernel.triton.from; -const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; +const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +41,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. @@ -59,7 +62,7 @@ pub fn fusedExpertsBase( gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); @@ -141,7 +144,7 @@ pub fn fusedExpertsBase( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -490,8 +493,8 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { - return zml.moe.applyActivation(x, activation, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); @@ -506,7 +509,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -794,7 +797,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); @@ -899,7 +902,7 @@ pub fn fusedExpertsMxfp4( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} From d7debcb3ff14914b9c0f23e619e93e3cc97e188c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sun, 8 Nov 2026 21:51:58 +0000 Subject: [PATCH 5/6] zml/kernels/moe: break cycle between flashinfer_cutlass and platform * Stopped using `zml.ops.CustomCall` because it creates cycles impossible to break. * renamed some `cutlass_flashinfer` to `flashinfer_cutlass` for consistency I didn't replace `CustomCall` because flashinfer_cutlass is the last user and will be removed soon. --- zml/BUILD.bazel | 2 +- .../moe/cutlass_flashinfer/BUILD.bazel | 8 - .../moe/flashinfer_cutlass/BUILD.bazel | 10 + .../moe/flashinfer_cutlass/contract.zig | 40 +++ .../flashinfer_cutlass.zig} | 244 ++---------------- .../moe/flashinfer_cutlass/runtime.zig | 223 ++++++++++++++++ zml/kernels/moe/moe.zig | 12 +- zml/platform.zig | 44 +++- 8 files changed, 336 insertions(+), 247 deletions(-) delete mode 100644 zml/kernels/moe/cutlass_flashinfer/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/contract.zig rename zml/kernels/moe/{cutlass_flashinfer/cutlass_flashinfer.zig => flashinfer_cutlass/flashinfer_cutlass.zig} (59%) create mode 100644 zml/kernels/moe/flashinfer_cutlass/runtime.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 3c0ace0b..fff3b8a8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -119,7 +119,7 @@ zig_library( "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/cutlass_flashinfer", + "//zml/kernels/moe/flashinfer_cutlass", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel deleted file mode 100644 index 54b8604c..00000000 --- a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cutlass_flashinfer", - srcs = [ - "cutlass_flashinfer.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig similarity index 59% rename from zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig rename to zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 118593a8..d1a0e1b5 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,12 +5,13 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; const GlobalOptions = @import("../options.zig").Options; const Activation = @import("../options.zig").Activation; const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const runtime = @import("runtime.zig"); -const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); pub const auto_tactic: i32 = -1; @@ -45,112 +46,15 @@ pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); } -const Input = struct { - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, -}; +const Inputs = contract.Inputs(zml.Tensor); -const Output = struct { - output: zml.Shape, - workspace: zml.Shape, -}; +const Outputs = contract.Outputs(zml.Shape); -const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; +const Attributes = contract.Attributes; -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -fn runnersFromAttributes(attributes: Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -fn currentRunners() !*Runners { +fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.fi_cutlass_moe_runners; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; return runners orelse error.RunnersNotLoaded; } @@ -166,80 +70,22 @@ fn cutlassActivation(activation: Activation) i32 { }; } -fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; -fn ffiCallNvfp4( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, -) !?*zml.pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = input.hidden_states.ptr; - io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); - io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); - io.fc1_expert_weights = input.fc1_weights.ptr; - io.fc2_expert_weights = input.fc2_weights.ptr; - io.quant_scales[0] = input.fc1_act_global.ptr; - io.quant_scales[1] = input.fc1_weight_block.ptr; - io.quant_scales[2] = input.fc1_global.ptr; - io.quant_scales[3] = input.fc2_act_global.ptr; - io.quant_scales[4] = input.fc2_weight_block.ptr; - io.quant_scales[5] = input.fc2_global.ptr; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = output.output.ptr; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = output.workspace.ptr; - workspace.data_bytes = output.workspace.shape.byteSize(); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, ); - return null; } -const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ - .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert mesh is owned by forwardMoe outer manual computation. - .sharding_aware = false, - .has_side_effect = false, -}); - fn computeCapability(platform: *const zml.Platform) !u16 { const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; return switch (cc.sm()) { @@ -248,40 +94,15 @@ fn computeCapability(platform: *const zml.Platform) !u16 { }; } -pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, -) !void { - if (comptime platforms.isEnabled(.cuda)) { - var cuda_state = &platform.state.cuda; - if (cuda_state.fi_cutlass_moe_runners != null) return; - try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); - const runners = try allocator.create(Runners); - runners.* = .{}; - cuda_state.fi_cutlass_moe_runners = runners; - return; - } - return error.UnsupportedPlatform; -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try routedNvfp4Call.register(platform); - return; - } - return error.UnsupportedPlatform; -} - pub fn isAvailable(platform: *const zml.Platform) bool { if (platform.target != .cuda) return false; - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; _ = computeCapability(platform) catch return false; return true; } pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; const compute_capability = computeCapability(platform) catch return false; return switch (compute_capability) { 100, 103, 120 => true, @@ -289,21 +110,6 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { }; } -pub fn tacticCounts( - platform: *const zml.Platform, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - fn roundUp(value: i64, alignment: i64) i64 { return @divTrunc(value + alignment - 1, alignment) * alignment; } @@ -468,10 +274,10 @@ pub fn fusedExpertsNvfp4( attributes.runners = @intFromPtr(runners); const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = makeContext(attributes); + const context = runtime.makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - checkStatus( + runtime.checkStatus( deviceRunner.api, deviceRunner.api.getWorkspaceRequirements( deviceRunner.runner, @@ -480,7 +286,7 @@ pub fn fusedExpertsNvfp4( ), ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - const result = routedNvfp4Call.call( + const result = call( .{ .hidden_states = hidden_states, .fc1_weights = fc1_weights, diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index b3ca9814..32bebac0 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -5,11 +5,11 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; pub const cute = @import("cute/cute.zig"); -pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const ExpertsLayout = @import("options.zig").ExpertsLayout; const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); @@ -45,7 +45,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (cutlass_flashinfer.isNvfp4Supported(platform)) + else if (flashinfer_cutlass.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -84,7 +84,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -117,7 +117,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => cutlass_flashinfer.register(platform), + .flashinfer_cutlass => flashinfer_cutlass.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -262,7 +262,7 @@ pub fn forwardMoe( const output = switch (args.backend) { .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..f3510a19 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -8,6 +8,8 @@ pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -257,12 +259,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, fn deinit(self: *CudaState) void { - if (self.fi_cutlass_moe_runners) |runners| { + if (self.flashinfer_cutlass_moe_runners) |runners| { runners.deinit(); - self.fi_cutlass_moe_runners = null; + self.flashinfer_cutlass_moe_runners = null; } } }; @@ -381,16 +383,9 @@ pub const Platform = struct { switch (target) { .cuda => { - if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { - zml.moe.cutlass_flashinfer.register(platform) catch |err| { - log.warn( - "Failed to register FlashInfer CUTLASS MoE custom calls: {}", - .{err}, - ); - }; - } else |err| { - log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); - } + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + }; }, else => {}, } @@ -877,6 +872,29 @@ pub const cuda = struct { } } else null; } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } }; pub const rocm = struct { From 7283f5ce0256eb561b413b6001b2d2e27f3b43ac Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sun, 8 Nov 2026 21:57:30 +0000 Subject: [PATCH 6/6] zml: remove obsolete code related to custom calls It will come back under a better abstraction if required --- zml/ops.zig | 116 -------------------------------------------------- zml/pjrtx.zig | 8 ---- 2 files changed, 124 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index 4a189035..10d15add 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -23,10 +23,8 @@ pub const manualComputation2 = manual_computation2.manualComputation2; pub const ManualPartitioning = manual_computation2.ManualPartitioning; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; -const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; @@ -2733,85 +2731,6 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value }; } -pub fn CustomCall( - // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } - I: type, - O: type, - A: type, - // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O - // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` - // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error - comptime func_: anytype, - comptime params: struct { - /// Name of the custom call, must be unique. - name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, - /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. - has_side_effect: bool, - /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. - /// Use the input/output field names to express the mapping: `{ . = . }` - output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, - /// Request XLA host-compute offload for this custom call on accelerator backends. - /// Input/Output buffers will automatically be moved d2h/h2d without the need for an explicit `.toMemory(...)`. - compute_on_host: bool = false, - }, -) type { - return struct { - pub const InputTensors = I; - pub const OutputShapes = O; - pub const Attributes = A; - - /// Register the custom call so you can use it. - pub fn register(platform: *const Platform) !void { - try platform.registerFfi(.{ - .name = params.name, - .platform_name = if (params.compute_on_host) "Host" else null, - .handler = @This().handler, - .traits = .{ - .command_buffer_compatible = true, - }, - }); - } - - pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { - const opts: CustomCallOptions = .{ - .has_side_effect = params.has_side_effect, - .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), - .compute_on_host = params.compute_on_host, - }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } - } - - fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); - return func_(call_frame, input_buffers, output_buffers, attributes) catch |err| - pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); - } - }; -} - pub fn ShapeToTensor(T: type) type { return meta.MapRestrict(Shape, Tensor).map(T); } @@ -2995,41 +2914,6 @@ pub fn typedCustomCall( } } -fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { - TensorToCustomCallBuffer(I), - ShapeToCustomCallBuffer(O), - A, -} { - var input: TensorToCustomCallBuffer(I) = undefined; - inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.args.buffers()[i]; - @field(input, field_name) = .fromPjrt(buf); - } - - var output: ShapeToCustomCallBuffer(O) = undefined; - inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { - const shape_buf = call_frame.results.buffers()[i]; - @field(output, field_name) = .fromPjrt(shape_buf); - } - - const attributes: A = switch (@typeInfo(A)) { - .void => {}, - .@"struct" => blk: { - var attrs: A = undefined; - inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { - // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); - } - break :blk attrs; - }, - else => @compileError("Unsupported attributes type: " ++ @typeName(A)), - }; - - return .{ input, output, attributes }; -} - test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 824d2bd1..8887ad17 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -45,14 +45,6 @@ pub const Client = opaque { } }; -pub fn TensorToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); -} - -pub fn ShapeToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Shape, CustomCallBuffer).map(T); -} - pub const CustomCallBuffer = struct { shape: Shape, ptr: *anyopaque,