From 354fcd68466bc76353505421c9aedb3013051e97 Mon Sep 17 00:00:00 2001 From: dmehala Date: Sat, 14 Nov 2026 10:26:05 +0000 Subject: [PATCH] wip --- zml/kernels/paged_attention/cute/BUILD.bazel | 16 ++ .../paged_attention/cute/benchmark.zig | 28 +- zml/kernels/paged_attention/cute/cases.zig | 28 +- .../paged_attention/cute/correctness_test.zig | 2 + zml/kernels/paged_attention/cute/cute.zig | 2 +- zml/kernels/paged_attention/cute/fixtures.zig | 20 +- .../paged_attention/cute/reference.zig | 11 +- .../paged_attention/cute/sm10x/attention.zig | 272 +++++++++++++++--- .../paged_attention/cute/sm10x/sm10x.zig | 31 +- .../paged_attention/cute/sparse_benchmark.zig | 14 + 10 files changed, 358 insertions(+), 66 deletions(-) create mode 100644 zml/kernels/paged_attention/cute/sparse_benchmark.zig diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index 6329ae12..000e975b 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -45,3 +45,19 @@ zig_binary( }), deps = ["//zml"], ) + +zig_binary( + name = "sparse_benchmark", + srcs = [ + "benchmark.zig", + "cases.zig", + "fixtures.zig", + ], + main = "sparse_benchmark.zig", + tags = ["manual"], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 587af69f..a509f8c4 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -5,6 +5,10 @@ const fixtures = @import("fixtures.zig"); const Inputs = fixtures.Inputs; pub fn main(init: std.process.Init) !void { + try run(init, &cases.benchmarks, 1); +} + +pub fn run(init: std.process.Init, benchmark_cases: []const cases.Case, chain: usize) !void { const platform = try zml.Platform.auto(init.gpa, init.io, .{}); defer platform.deinit(init.gpa, init.io); @@ -19,9 +23,9 @@ pub fn main(init: std.process.Init) !void { platform.fmtVerbose(), }); - std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); + std.debug.print("timing=synchronized_chain_us_per_attention chain={d} devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{ chain, platform.devices.len }); var failures: usize = 0; - for (cases.benchmarks) |c| { + for (benchmark_cases) |c| { std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ c.name, c.head_dim, @@ -34,7 +38,7 @@ pub fn main(init: std.process.Init) !void { @tagName(c.mask), }); - runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { + runBenchmark(init.gpa, init.io, platform, mesh, c, chain) catch |err| { failures += 1; std.debug.print("FAIL error={s}\n", .{@errorName(err)}); }; @@ -43,7 +47,7 @@ pub fn main(init: std.process.Init) !void { if (failures != 0) return error.BenchmarkFailed; } -fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case, chain: usize) !void { // Keep number of head per GPU constant. var global_case = c; global_case.kv_heads *= platform.devices.len; @@ -54,7 +58,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, chain }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -86,8 +90,8 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { const call_start: std.Io.Timestamp = .now(io, .awake); runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - samples[sample_count] = elapsedUs(call_start, io); - accumulated_us += samples[sample_count]; + samples[sample_count] = elapsedUs(call_start, io) / @as(f64, @floatFromInt(chain)); + accumulated_us += samples[sample_count] * @as(f64, @floatFromInt(chain)); sample_count += 1; output.deinit(); } @@ -115,17 +119,21 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, chain: usize) zml.Tensor { const Call = struct { inputs: Inputs, case: cases.Case, + chain: usize, fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forward(self.inputs, self.case); + // Keep the calls dependent so graph CSE cannot remove repetitions. + var current = self.inputs; + for (0..self.chain) |_| current.q = fixtures.forward(current, self.case); + return current.q; } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c, .chain = chain }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index e8dedeaa..050e9473 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -15,16 +15,24 @@ pub const Case = struct { kv_len: usize = 65, kv_capacity: ?usize = null, inactive: usize = 0, + leading_inactive: bool = false, + padding: usize = 0, shuffle_pages: bool = false, + random_indices: bool = false, + empty_key_tiles: bool = false, + padded_selection: bool = false, skew: bool = false, mask: Mask = .causal, empty_indices: bool = false, + selection_count: usize = topk, window: u32 = 17, sink: bool = false, scale: ?f32 = null, pub fn tokens(c: Case) usize { - return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; + var count = c.padding; + for (0..c.batch) |seq| count += c.queryLen(seq); + return count; } pub fn pages(c: Case) usize { return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; @@ -33,7 +41,7 @@ pub const Case = struct { return c.kv_heads * c.group; } pub fn queryLen(c: Case, seq: usize) usize { - return if (seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; + return if (if (c.leading_inactive) seq < c.inactive else seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; } pub fn kvLen(c: Case, seq: usize) usize { const qlen = c.queryLen(seq); @@ -45,7 +53,7 @@ pub const Case = struct { // Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. pub const correctness = blk: { - var result: [34]Case = undefined; + var result: [48]Case = undefined; var i: usize = 0; for ([_]usize{ 64, 128, 256, 512 }) |dim| { for ([_]bool{ false, true }) |mixed| { @@ -79,6 +87,20 @@ pub const correctness = blk: { result[31] = .{ .name = "zero_scale", .head_dim = 256, .prefill = 1, .kv_len = 257, .sink = true, .scale = 0 }; result[32] = .{ .name = "d512_window_prefill", .head_dim = 512, .batch = 1, .prefill = 1, .chunk = 33, .kv_len = 513, .mask = .window, .window = 129, .sink = true }; result[33] = .{ .name = "prefill_inactive", .head_dim = 64, .batch = 4, .prefill = 2, .inactive = 2, .kv_len = 257, .group = 5 }; + result[34] = .{ .name = "sparse_decode", .head_dim = 256, .batch = 16, .kv_heads = 1, .group = 12, .kv_len = 4097, .mask = .indices, .selection_count = 2048, .shuffle_pages = true }; + result[35] = .{ .name = "sparse_tail", .head_dim = 32, .mask = .indices, .selection_count = 129 }; + result[36] = .{ .name = "sparse_mixed_sink", .head_dim = 512, .prefill = 1, .mask = .indices, .selection_count = 129, .sink = true }; + result[37] = .{ .name = "sparse_empty", .mask = .indices, .selection_count = 129, .empty_indices = true }; + result[38] = .{ .name = "sparse_empty_sink", .mask = .indices, .selection_count = 129, .empty_indices = true, .sink = true }; + result[39] = .{ .name = "sparse_inactive", .head_dim = 64, .batch = 4, .inactive = 2, .mask = .indices, .selection_count = 65 }; + result[40] = .{ .name = "sparse_prefill_group_tail", .head_dim = 256, .group = 19, .batch = 4, .prefill = 2, .chunk = 5, .inactive = 1, .mask = .indices, .selection_count = 257, .sink = true, .shuffle_pages = true }; + result[41] = .{ .name = "sparse_prefill_large", .head_dim = 256, .group = 12, .batch = 2, .prefill = 2, .chunk = 65, .kv_len = 2049, .mask = .indices, .selection_count = 2048, .shuffle_pages = true }; + result[42] = .{ .name = "sparse_batch_tail", .head_dim = 64, .group = 3, .batch = 40, .prefill = 1, .chunk = 2, .inactive = 3, .mask = .indices, .selection_count = 257, .random_indices = true }; + result[43] = .{ .name = "sparse_prefill_padding", .head_dim = 256, .group = 12, .batch = 2, .prefill = 1, .chunk = 3, .padding = 7, .mask = .indices }; + result[44] = .{ .name = "sparse_split_padding", .head_dim = 256, .group = 19, .batch = 2, .prefill = 1, .chunk = 3, .padding = 7, .mask = .indices, .selection_count = 257 }; + result[45] = .{ .name = "sparse_leading_inactive", .head_dim = 256, .group = 12, .batch = 8, .inactive = 3, .leading_inactive = true, .padding = 3, .mask = .indices, .selection_count = 257 }; + result[46] = .{ .name = "sparse_empty_key_tiles", .head_dim = 256, .group = 12, .batch = 2, .prefill = 1, .chunk = 5, .padding = 2, .mask = .indices, .selection_count = 2051, .random_indices = true, .empty_key_tiles = true }; + result[47] = .{ .name = "sparse_all_empty_key_tiles", .head_dim = 256, .group = 12, .batch = 2, .prefill = 1, .chunk = 5, .padding = 2, .mask = .indices, .selection_count = 2051, .empty_indices = true, .sink = true }; break :blk result; }; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index 436b9d3a..7fea075e 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -44,6 +44,8 @@ test "FP16 paged attention" { .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, + .{ .name = "fp16_sparse_sink", .head_dim = 256, .prefill = 1, .mask = .indices, .selection_count = 129, .sink = true }, + .{ .name = "fp16_sparse_decode", .head_dim = 256, .mask = .indices, .selection_count = 129, .sink = true }, }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ef78cc49..795fcd7d 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -68,7 +68,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); - if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + if (sm10x.isAvailable(zml.Compiler.current().platform)) return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index c0815d1d..1baf9ccf 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -24,7 +24,7 @@ pub const Fixture = struct { .lengths = try a.alloc(i32, c.batch), .starts = try a.alloc(i32, c.batch + 1), .sinks = try a.alloc(f32, c.heads()), - .indices = try a.alloc(i32, c.tokens() * cases.topk), + .indices = try a.alloc(i32, c.tokens() * c.selection_count), }; var rng = std.Random.DefaultPrng.init(42); const random = rng.random(); @@ -55,7 +55,21 @@ pub const Fixture = struct { } for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { const l: i32 = @intCast(len); - @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); + const pattern = [_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }; + for (f.indices[t * c.selection_count ..][0..c.selection_count], 0..) |*index, j| { + index.* = if (c.random_indices and j % pattern.len != 3 and j % pattern.len != 4 and j % pattern.len != 5) + @intCast(random.uintLessThan(usize, len)) + else if (j % pattern.len == 0) + @mod(@as(i32, @intCast(t)) - f.starts[s], l) + else + pattern[j % pattern.len]; + if (c.empty_key_tiles and (j / 128) % 2 == 1) index.* = -1; + if (c.padded_selection) { + const visible: i32 = @intCast(len - c.queryLen(s) + t - @as(usize, @intCast(f.starts[s])) + 1); + const tail = c.selection_count - 3; + index.* = if (j < @as(usize, @intCast(@max(0, visible - 3)))) @intCast(j) else if (j >= tail) visible - 3 + @as(i32, @intCast(j - tail)) else -1; + } + } } } if (c.empty_indices) { @@ -91,7 +105,7 @@ pub const Inputs = struct { .lengths = .init(.{ .b = c.batch }, .i32), .starts = .init(.{ .b = c.batch + 1 }, .i32), .sinks = .init(.{c.heads()}, .f32), - .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), + .indices = .init(.{ .b = c.tokens(), .topk = c.selection_count }, .i32), }; } diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig index 1a030646..9a0d17d3 100644 --- a/zml/kernels/paged_attention/cute/reference.zig +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -10,7 +10,7 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u const start: usize = @intCast(f.starts[seq]); const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); const position = len - qlen + token - start; - const count = if (c.mask == .indices) cases.topk else len; + const count = if (c.mask == .indices) c.selection_count else len; const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); @@ -44,7 +44,7 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u } fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { - return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); + return if (c.mask == .indices) f.indices[token * c.selection_count + j] else @intCast(j); } pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { @@ -52,7 +52,12 @@ pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: [ for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; } - const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); + if (c.mask == .indices) { + for (actual[active_tokens * c.heads() * c.head_dim ..]) |v| { + if (v.toF32() != 0) return error.NonzeroQueryPadding; + } + } + const scores = try a.alloc(f32, @max(c.kv_len, c.selection_count)); defer a.free(scores); const expected = try a.alloc(f32, c.head_dim); defer a.free(expected); diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 12d6273c..3f43f8b7 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -19,12 +19,20 @@ pub const Config = struct { page_size: i64, max_query: i64, dtype: cute.DType, + // Transposed tensor-core tile; sparse prefill also uses it per query. decode: bool, causal: bool, window: i64, scale: f32, sink: bool, splits: i64 = 1, + indexed: bool = false, + topk: i64 = 0, + compact: bool = false, + + pub fn selectionTiles(c: Config) i64 { + return @divTrunc(c.topk + c.keys() - 1, c.keys()); + } pub fn rows(c: Config) i64 { return if (c.decode) (if (c.group <= 8) 8 else 16) else 64; @@ -79,7 +87,7 @@ pub const Config = struct { pub const block: [3]i32 = .{ 256, 1, 1 }; pub const Attention = cute.Kernel(Config, .{ .name = "paged_attention_cute", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices", "tiles" }, .outputs = &.{"out"}, .run = attention, }); @@ -88,6 +96,26 @@ fn loadInt(b: *B, pointer: V, index: V) V { return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; } +// Coalesced starts-table loads and a warp reduction replace a dependent +// per-sequence scan. All lanes participate after the masked load reconverges. +fn findSequence(b: *B, c: Config, starts: V, token: V) V { + const lane = b.threadIdx().x.rem(32); + var batches = b.openFor(0, c.batch, 32, .{b.constant(.i32, 0)}); + { + const candidate = batches.iv.add(lane); + var entry = b.openIfElse(candidate.lt(c.batch), .{b.constant(.i32, 0)}); + { + const owns = token.ge(loadInt(b, starts, candidate)).bitAnd(token.lt(loadInt(b, starts, candidate.add(1)))); + entry.yieldThen(.{b.select(owns, candidate, b.constant(.i32, 0))}); + } + { + entry.yieldElse(.{b.constant(.i32, 0)}); + } + batches.yield(.{batches.carried[0].maximum(b.warpMaxNonNegative(entry.results[0]))}); + } + return batches.results[0]; +} + // Each 64-feature plane uses a 128-byte XOR swizzle. The descriptor and // vector copies use this same layout for Q, K, V and probabilities. fn swizzle(row: V, col: V, rows: i64) V { @@ -112,7 +140,8 @@ const Range = struct { first: V, count: V }; // Used by both attention and merge: every output row must merge exactly the // partitions produced for its query tile, including causal/window boundaries. -fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { +fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V, selected_tiles: V) Range { + if (c.indexed) return .{ .first = b.constant(.i32, 0), .count = if (c.compact) selected_tiles else b.constant(.i32, c.selectionTiles()) }; const first_pos = klen.sub(qlen).add(first_row.div(c.group)); const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); @@ -122,7 +151,8 @@ fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { } fn liveSplits(c: Config, tiles: V) V { - return tiles.add(1).div(2).maximum(1).minimum(c.splits); + const min_tiles: i64 = if (c.indexed) 1 else 2; + return tiles.add(min_tiles - 1).div(min_tiles).maximum(1).minimum(c.splits); } const Tile = struct { @@ -132,6 +162,8 @@ const Tile = struct { k: V, v: V, table: V, + indices: V, + tile_map: V, sink: V, out: V, shared: V, @@ -158,9 +190,33 @@ const Tile = struct { fn phase(t: Tile, id: V) V { return id.div(t.c.buffers()).rem(2); } + fn keyTile(t: Tile, compact_index: V) V { + if (!t.c.compact) return compact_index; + const token = t.start.add(t.first_row.div(t.c.group)); + return loadInt(t.b, t.tile_map, token.mul(t.c.selectionTiles() + 1).add(compact_index).add(1)); + } + // Each sparse tile owns one query and reuses its selection across GQA + // heads. Keep duplicates as independent entries; never dereference a + // selection-list tail or use an invalid index to load the page table. + fn logicalKey(t: Tile, slot: V) V { + if (!t.c.indexed) return slot; + const b = t.b; + var selected = b.openIfElse(slot.lt(t.c.topk), .{b.constant(.i32, 0)}); + { + selected.yieldThen(.{loadInt(b, t.indices, t.start.add(t.first_row.div(t.c.group)).mul(t.c.topk).add(slot))}); + } + { + selected.yieldElse(.{b.constant(.i32, -1)}); + } + return selected.results[0]; + } + fn validRow(t: Tile, row: V) V { + const active = row.lt(t.qlen.mul(t.c.group)); + return if (t.c.indexed) active.bitAnd(row.div(t.c.group).eq(t.first_row.div(t.c.group))) else active; + } fn valid(t: Tile, row: V, key: V) V { const pos = t.klen.sub(t.qlen).add(row.div(t.c.group)); - var result = row.lt(t.qlen.mul(t.c.group)).bitAnd(key.lt(t.klen)); + var result = t.validRow(row).bitAnd(key.ge(0)).bitAnd(key.lt(t.klen)); if (t.c.causal) result = result.bitAnd(key.le(pos)); if (t.c.window > 0) result = result.bitAnd(key.gt(pos.sub(t.c.window))); return result; @@ -199,19 +255,48 @@ fn attention(b: *B, c: Config) cute.FinishError!void { .lengths = .{ .ptr = cute.DType.i32 }, .starts = .{ .ptr = cute.DType.i32 }, .sink = .{ .ptr = cute.DType.f32 }, + .indices = .{ .ptr = cute.DType.i32 }, + .tiles = .{ .ptr = cute.DType.i32 }, .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, }); std.debug.assert(c.sharedBytes() <= 227 * 1024); b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); - const seq = b.blockIdx().x.div(c.queryTiles()); - const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const starts = b.ptrToInt(a.starts, .i64); + const packed_queries = c.indexed; + const head_tiles = @divTrunc(c.group + c.rows() - 1, c.rows()); + const packed_token = b.blockIdx().x.div(head_tiles); + // Sparse attention flattens packed queries instead of launching max_query + // tiles for every sequence. Each tile belongs to exactly one query. + const seq = if (packed_queries) findSequence(b, c, starts, packed_token) else b.blockIdx().x.div(c.queryTiles()); const start = loadInt(b, starts, seq); + const first_row = if (packed_queries) packed_token.sub(start).mul(c.group).add(b.blockIdx().x.rem(head_tiles).mul(c.rows())) else b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const qlen = loadInt(b, starts, seq.add(1)).sub(start); const klen = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); - const range = keyRange(b, c, first_row, qlen, klen); + const tile_map = b.ptrToInt(a.tiles, .i64); + const selected_tiles = if (c.compact) loadInt(b, tile_map, packed_token.mul(c.selectionTiles() + 1)) else b.constant(.i32, 0); + const range = keyRange(b, c, first_row, qlen, klen, selected_tiles); const splits = liveSplits(c, range.count); const split = b.blockIdx().z; + const output = b.ptrToInt(a.out, .i64); + if (c.indexed and c.splits == 1) { + // Define unused query capacity just as the generic sparse kernel does. + // Active and padded rows are disjoint, including holes in sequences. + var unused = b.openIf(packed_token.ge(loadInt(b, starts, b.constant(.i32, c.batch)))); + { + var dims = b.openFor(b.threadIdx().x, c.rows() * c.head_dim, 256, .{}); + { + const head = b.blockIdx().x.rem(head_tiles).mul(c.rows()).add(dims.iv.div(c.head_dim)); + var owns = b.openIf(head.lt(c.group)); + { + const index = packed_token.to(.i64).mul(c.kv_heads).add(b.blockIdx().y.to(.i64)).mul(c.group).add(head.to(.i64)).mul(c.head_dim).add(dims.iv.rem(c.head_dim).to(.i64)); + b.storeHalf(output.add(index.mul(2)), .global, b.constant(.i32, 0)); + owns.close(); + } + dims.close(); + } + unused.close(); + } + } var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); { const tid = b.threadIdx().x; @@ -225,8 +310,10 @@ fn attention(b: *B, c: Config) cute.FinishError!void { .k = b.ptrToInt(a.k, .i64), .v = b.ptrToInt(a.v, .i64), .table = b.ptrToInt(a.table, .i64), + .indices = b.ptrToInt(a.indices, .i64), + .tile_map = tile_map, .sink = b.ptrToInt(a.sink, .i64), - .out = b.ptrToInt(a.out, .i64), + .out = output, .shared = shared, .tmem = undefined, .tid = tid, @@ -263,7 +350,7 @@ fn attention(b: *B, c: Config) cute.FinishError!void { const row = queries.iv.div(@divExact(c.paddedDim(), 8)); const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); const packed_row = first_row.add(row); - const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); + const valid = t.validRow(packed_row).bitAnd(col.lt(c.head_dim)); const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); @@ -304,13 +391,24 @@ fn produce(t: Tile) void { var tiles = b.openFor(0, t.tiles, 1, .{}); { // Paged offsets are shared by K/V and every 128-feature stage. + const key_tile = t.keyTile(t.first_key_tile.add(tiles.iv)); var offsets: [16]V = undefined; var valid: [16]V = undefined; for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { - const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(i * 8); - valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); - const safe = b.select(key.lt(t.klen), key, b.constant(.i32, 0)); - const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); + const key = t.logicalKey(key_tile.mul(c.keys()).add(row0).add(i * 8)); + const live = key.ge(0).bitAnd(key.lt(t.klen)); + valid[i] = live.bitAnd(col.lt(c.head_dim)); + const safe = b.select(live, key, b.constant(.i32, 0)); + const page = if (c.indexed) blk: { + var lookup = b.openIfElse(live, .{b.constant(.i32, 0)}); + { + lookup.yieldThen(.{loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size)))}); + } + { + lookup.yieldElse(.{b.constant(.i32, 0)}); + } + break :blk lookup.results[0]; + } else loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); } for (0..@as(usize, @intCast(c.features() * 2))) |stage| { @@ -384,9 +482,14 @@ fn reduceRow(comptime n: usize, values: [n]V, comptime maximum: bool) V { fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximum: bool) [heads]V { const b = t.b; - for (values, 0..) |value, h| { + const active_heads: usize = @intCast(@min(t.c.group, heads)); + for (values[0..active_heads], 0..) |value, h| { var reduced = value; - inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); + if (maximum) { + reduced = b.warpMaxF32(reduced); + } else { + inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = reduced.add(b.shuffleXor(reduced, step)); + } var leader = b.openIf(t.lane.eq(0)); { b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(h * 4), .shared, .{reduced.bitCast(.i32)}); @@ -394,8 +497,8 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu } } b.barrierCtaSyncAligned(1, 128); - var result: [heads]V = undefined; - for (0..heads) |h| { + var result: [heads]V = @splat(b.constant(.f32, if (maximum) -std.math.inf(f32) else 0)); + for (0..active_heads) |h| { var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); for (1..4) |w| { const next = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); @@ -410,6 +513,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu fn decode(t: Tile, comptime heads: usize) void { const b = t.b; const c = t.c; + const active_heads: usize = @intCast(@min(c.group, heads)); const address = t.tmem.add(t.warp.mul(32).shl(16)); const key_row = t.warp.mul(32).add(t.lane); const zero = b.constant(.f32, 0); @@ -424,32 +528,32 @@ fn decode(t: Tile, comptime heads: usize) void { multiply(t, tiles.iv, true); const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); b.fenceTmemLoad(); - const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); - var scores: [heads]V = undefined; - for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); + const key = t.logicalKey(t.keyTile(t.first_key_tile.add(tiles.iv)).mul(c.keys()).add(key_row)); + var scores: [heads]V = @splat(neg_inf); + for (0..active_heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); const maxima = reduceHeads(t, heads, scores, true); - var next: [2 * heads]V = undefined; - var alpha: [heads]V = undefined; - var probabilities: [heads]V = undefined; - for (0..heads) |h| { + var next = initial; + var alpha: [heads]V = @splat(zero); + var probabilities: [heads]V = @splat(zero); + for (0..active_heads) |h| { next[h] = tiles.carried[h].maximum(maxima[h]); const reference = next[h].maximum(-std.math.floatMax(f32)); alpha[h] = b.exp2Approx(tiles.carried[h].sub(reference)); probabilities[h] = b.exp2Approx(scores[h].sub(reference)); } const sums = reduceHeads(t, heads, probabilities, false); - for (0..heads) |h| { + for (0..active_heads) |h| { next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); t.putProbability(b.constant(.i32, h), key_row, probabilities[h]); } - for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); + for (active_heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); var rescale = b.openIf(tiles.iv.gt(0)); { for (0..@as(usize, @intCast(c.features()))) |part| { const addr = address.add(@as(i64, @intCast(part)) * c.rows()); var out = b.tmemLoad32x32b(heads, addr); b.fenceTmemLoad(); - for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + for (0..active_heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); b.tmemStore32x32b(heads, addr, out); } b.fenceTmemStore(); @@ -463,9 +567,9 @@ fn decode(t: Tile, comptime heads: usize) void { const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); b.fenceTmemLoad(); const dim = key_row.add(part * 128); - for (0..heads) |h| { + for (0..active_heads) |h| { const row = t.first_row.add(h); - var valid = b.openIf(row.lt(t.qlen.mul(c.group))); + var valid = b.openIf(t.validRow(row)); { var feature = b.openIf(dim.lt(c.head_dim)); { @@ -599,7 +703,7 @@ fn prefill(t: Tile) void { pub const Merge = cute.Kernel(Config, .{ .name = "paged_attention_cute_merge", - .inputs = &.{ "partials", "lengths", "starts", "sink" }, + .inputs = &.{ "partials", "lengths", "starts", "sink", "tiles" }, .outputs = &.{"out"}, .run = merge, }); @@ -610,22 +714,32 @@ fn merge(b: *B, c: Config) cute.FinishError!void { .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{if (c.sink) c.kv_heads * c.group else 1} } }, + .tiles = .{ .ptr = cute.DType.i32 }, .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, }); const token = b.blockIdx().x; const head = b.blockIdx().y; - var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); - { - var find = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); + const total = a.starts.get(.{c.batch}); + if (c.indexed) { + var unused = b.openIf(token.ge(total)); { - const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); - find.yield(.{b.select(owns, find.iv, find.carried[0])}); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + a.out.set(.{ token, head, dims.iv }, b.constant(.f32, 0).to(c.dtype)); + dims.close(); + } + unused.close(); } - const seq = find.results[0]; + } + var active = b.openIf(token.lt(total)); + { + const starts = b.ptrToInt(b.getIter(a.starts, b.ptrTy(.i32, .gmem, 16) catch unreachable), .i64); + const seq = findSequence(b, c, starts, token); const start = a.starts.get(.{seq}); const qlen = a.starts.get(.{seq.add(1)}).sub(start); const row = token.sub(start).mul(c.group).add(head.rem(c.group)); - const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); + const selected_tiles = if (c.compact) loadInt(b, b.ptrToInt(a.tiles, .i64), token.mul(c.selectionTiles() + 1)) else b.constant(.i32, 0); + const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq}), selected_tiles); const splits = liveSplits(c, range.count); const zero = b.constant(.f32, 0); const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); @@ -665,3 +779,87 @@ test "SM10x layouts and instruction feature gates" { defer std.testing.allocator.free(merge_ir); }; } + +// Compact nonempty selection tiles once per query, reusing the map across +// KV heads and split CTAs. Positions within each tile retain their order. +pub const TileMap = cute.Kernel(Config, .{ + .name = "paged_attention_cute_tile_map", + .inputs = &.{ "indices", "lengths", "starts" }, + .outputs = &.{"tiles"}, + .run = tileMap, +}); + +fn tileMap(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .indices = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .tiles = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.tokens, c.selectionTiles() + 1 } } }, + }); + const token = b.blockIdx().x; + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const starts = b.ptrToInt(a.starts, .i64); + var leader = b.openIf(tid.eq(0)); + a.tiles.set(.{ token, 0 }, b.constant(.i32, 0)); + leader.close(); + var active = b.openIf(token.lt(loadInt(b, starts, b.constant(.i32, c.batch)))); + { + const seq = findSequence(b, c, starts, token); + const len = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); + // Long contexts usually fill the list. Identity is always correct, + // including invalid entries, whose loads the attention kernel masks. + var identity = b.openIf(len.ge(c.topk)); + { + var entries = b.openFor(tid, c.selectionTiles(), 128, .{}); + a.tiles.set(.{ token, entries.iv.add(1) }, entries.iv); + entries.close(); + var first = b.openIf(tid.eq(0)); + a.tiles.set(.{ token, 0 }, b.constant(.i32, c.selectionTiles())); + first.close(); + identity.close(); + } + var sparse = b.openIf(len.lt(c.topk)); + { + const flags = b.ptrToInt(b.allocSmem(.i32, @max(1, c.selectionTiles()), 16), .i32); + const indices = b.ptrToInt(a.indices, .i64); + var tiles = b.openFor(tid.div(32), c.selectionTiles(), 4, .{}); + { + var valid = b.constant(.i32, 0); + for (0..4) |part| { + const slot = tiles.iv.mul(c.keys()).add(lane).add(part * 32); + var entry = b.openIfElse(slot.lt(c.topk), .{b.constant(.i32, 0)}); + { + const key = loadInt(b, indices, token.mul(c.topk).add(slot)); + entry.yieldThen(.{key.ge(0).bitAnd(key.lt(len)).to(.i32)}); + } + { + entry.yieldElse(.{b.constant(.i32, 0)}); + } + valid = valid.bitOr(entry.results[0]); + } + const any = b.warpMaxNonNegative(valid); + var first = b.openIf(lane.eq(0)); + b.storeWords(1, flags.add(tiles.iv.mul(4)), .shared, .{any}); + first.close(); + tiles.close(); + } + b.syncThreads(); + var first = b.openIf(tid.eq(0)); + { + var entries = b.openFor(0, c.selectionTiles(), 1, .{b.constant(.i32, 0)}); + { + const valid = b.loadWords(1, flags.add(entries.iv.mul(4)), .shared, .{})[0]; + var keep = b.openIf(valid.ne(0)); + a.tiles.set(.{ token, entries.carried[0].add(1) }, entries.iv); + keep.close(); + entries.yield(.{entries.carried[0].add(valid)}); + } + a.tiles.set(.{ token, 0 }, entries.results[0]); + first.close(); + } + sparse.close(); + } + active.close(); + } +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index afc4298e..e935312d 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -24,6 +24,8 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); const decode = !parameters.options_.is_prefill; + const indexed = opts.mask == .indices; + const indices = if (indexed) opts.mask.indices.transpose(.{ .b, .topk }) else zml.Tensor.scalar(0, .i32); const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; var c: kernel.Config = .{ .architecture = if (cc.minor >= 3) .sm103 else .sm100, @@ -36,23 +38,34 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .page_size = k_.dim(.k_chunk), .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), .dtype = zml.kernel.cute.from(q_.dtype()), - .decode = decode, + .decode = decode or indexed, + .indexed = indexed, + .topk = if (indexed) indices.dim(.topk) else 0, + .compact = indexed and !decode and indices.dim(.topk) > 128, .causal = opts.mask.isCausal(), .window = opts.mask.slidingWindow() orelse 0, .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), .sink = opts.sink != null, }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const query_tiles = if (indexed) c.tokens * @divTrunc(c.group + c.rows() - 1, c.rows()) else if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + const bound = if (indexed) c.topk else if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + const sms: i64 = @intCast(cute.multiprocessorCount(zml.Compiler.current().platform)); + const min_tiles: i64 = if (indexed) 1 else 2; + // Sparse decode benefits from one key tile per CTA. For larger query + // batches, keep the CTA count inside one two-CTA-per-SM wave. + const occupancy_splits = @divTrunc(sms * 2 + (if (indexed) @as(i64, 0) else work - 1), work); + c.splits = @max(1, @min(32, @min(occupancy_splits, @divTrunc(bound + min_tiles * c.keys() - 1, min_tiles * c.keys())))); // An unused scalar stays a constant buffer; broadcasting a dummy for // every head would add a helper kernel to each attention invocation. const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + const tiles = if (c.compact) kernel.TileMap.call(.{ .indices = indices, .lengths = parameters.seq_lens, .starts = parameters.query_start_len }, .{ + .tiles = zml.Shape.init(.{ c.tokens, c.selectionTiles() + 1 }, .i32), + }, .{ .cfg = c, .grid = .{ @intCast(c.tokens), 1, 1 }, .block = .{ 128, 1, 1 } }).tiles else zml.Tensor.scalar(0, .i32); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink, .indices = indices, .tiles = tiles }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + }, .{ .cfg = c, .grid = .{ @intCast(if (indexed) query_tiles else c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink, .tiles = tiles }, .{ .out = q_.shape() }, .{ .cfg = c, .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, .block = .{ 128, 1, 1 }, @@ -67,10 +80,10 @@ const ArchitectureCheck = struct { const lengths = zml.Tensor.constantTensor(.init(.{1}, .i32), std.mem.asBytes(&[_]i32{129})); const starts = zml.Tensor.constantTensor(.init(.{2}, .i32), std.mem.asBytes(&[_]i32{ 0, 3 })); const sink = zml.Tensor.zeroes(.init(.{3}, .f32)); - const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink }, .{ + const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink, .indices = zml.Tensor.scalar(0, .i32), .tiles = zml.Tensor.scalar(0, .i32) }, .{ .out = .init(.{ 2, 3, 3, c.head_dim + 2 }, .f32), }, .{ .cfg = c, .grid = .{ 1, 1, 2 }, .block = kernel.block }).out; - return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink }, .{ .out = q.shape() }, .{ + return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink, .tiles = zml.Tensor.scalar(0, .i32) }, .{ .out = q.shape() }, .{ .cfg = c, .grid = .{ 3, 3, 1 }, .block = .{ 128, 1, 1 }, diff --git a/zml/kernels/paged_attention/cute/sparse_benchmark.zig b/zml/kernels/paged_attention/cute/sparse_benchmark.zig new file mode 100644 index 00000000..bd758c41 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sparse_benchmark.zig @@ -0,0 +1,14 @@ +const std = @import("std"); + +const benchmark = @import("benchmark.zig"); +const cases = @import("cases.zig"); + +pub fn main(init: std.process.Init) !void { + try benchmark.run(init, &.{ + cases.Case{ .name = "qwen_decode", .head_dim = 256, .batch = 16, .kv_heads = 1, .group = 12, .kv_len = 4096, .mask = .indices, .selection_count = 2051, .random_indices = true, .shuffle_pages = true }, + cases.Case{ .name = "qwen_decode_single", .head_dim = 256, .batch = 1, .kv_heads = 1, .group = 12, .kv_len = 4096, .mask = .indices, .selection_count = 2051, .random_indices = true, .shuffle_pages = true }, + cases.Case{ .name = "qwen_prefill", .head_dim = 256, .batch = 2, .kv_heads = 1, .group = 12, .prefill = 2, .chunk = 64, .kv_len = 4096, .mask = .indices, .selection_count = 2051, .random_indices = true, .shuffle_pages = true }, + cases.Case{ .name = "qwen_mixed", .head_dim = 256, .batch = 16, .kv_heads = 1, .group = 12, .prefill = 4, .chunk = 64, .kv_len = 4096, .mask = .indices, .selection_count = 2051, .random_indices = true, .shuffle_pages = true }, + cases.Case{ .name = "qwen_prefill_short", .head_dim = 256, .batch = 2, .kv_heads = 1, .group = 12, .prefill = 2, .chunk = 64, .kv_len = 320, .mask = .indices, .selection_count = 2051, .padded_selection = true }, + }, 32); +}