diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel index 39d92259..aa9c1ed7 100644 --- a/zml/kernels/paged_attention/metal/BUILD.bazel +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -5,5 +5,7 @@ filegroup( srcs = [ "metal.zig", "paged_attn.zig", + "sparse_unified_attention.zig", + "sparse_unified_attention_test.zig", ], ) diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e2d68398..e4103435 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -6,11 +6,13 @@ const zml = @import("../../../zml.zig"); const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); +const sparse = @import("sparse_unified_attention.zig"); const log = std.log.scoped(.@"zml/attention/metal"); test { _ = paged_attn; + _ = @import("sparse_unified_attention_test.zig"); } pub const Options = struct { @@ -59,7 +61,7 @@ pub const Parameters = struct { }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); + if (opts.mask == .indices) return sparseAttention(parameters, q, k_cache, v_cache, opts); const num_kv_heads = q.dim(.hkv); @@ -211,3 +213,37 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } + +fn sparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + const indices = opts.mask.indices; + stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32 and indices.shape().hasTags(.{ .b, .topk }), "Sparse indices must be an i32 tensor tagged .b, .topk", .{}); + stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices must match queries and have nonempty capacity", .{}); + stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16, "Metal sparse attention requires bf16 or f16", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and q.dim(.hkv) == v.dim(.hkv) and q.dim(.hd) == k.dim(.hd) and q.dim(.hd) == v.dim(.hd), "Sparse attention requires matching query and KV heads and dimensions", .{}); + if (opts.sink) |sink| stdx.debug.assert(sink.count() == q.dim(.hkv) * q.dim(.hg), "Sparse attention requires one sink per query head", .{}); + + const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); + const out = sparse.call(.{ + .q = qh, + .k = k.transpose(.{ .page, .k_chunk, .hkv, .hd }), + .v = v.transpose(.{ .page, .k_chunk, .hkv, .hd }), + .indices = indices.transpose(.{ .b, .topk }), + .block_tables = parameters.block_table.transpose(.{ .b, .p }), + .seq_lens = parameters.seq_lens, + .query_starts = parameters.query_start_len, + .sinks = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }, qh.shape(), .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .num_heads = @intCast(qh.dim(.h)), + .num_queries_per_kv = @intCast(q.dim(.hg)), + .block_size = @intCast(k.dim(.k_chunk)), + .indices_count = @intCast(indices.dim(.topk)), + .max_blocks = @intCast(parameters.block_table.dim(.p)), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .use_sinks = opts.sink != null, + .num_segments = sparse.selectNumSegments(@intCast(indices.dim(.topk)), !parameters.options_.is_prefill), + }, opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))); + return out.splitAxis(.h, .{ .hkv = q.dim(.hkv), .hg = .auto }).transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/metal/sparse_unified_attention.zig b/zml/kernels/paged_attention/metal/sparse_unified_attention.zig new file mode 100644 index 00000000..1675e0cb --- /dev/null +++ b/zml/kernels/paged_attention/metal/sparse_unified_attention.zig @@ -0,0 +1,320 @@ +//! Sparse paged attention over selection slots, including repeated positions. +const std = @import("std"); +const metal = @import("../../../kernel.zig").metal; +const Builder = metal.Builder; +const zml = @import("../../../zml.zig"); + +const empty_max = -std.math.floatMax(f32) / 2; + +/// Partition selection slots, rather than logical cache positions. +pub const Segments = struct { + span: i32, + launched: i32, + + pub fn init(count: i32, capacity: i32) Segments { + std.debug.assert(count > 0 and capacity > 0); + const tiles = std.math.divCeil(i32, count, 32) catch unreachable; + const span = (std.math.divCeil(i32, tiles, capacity) catch unreachable) * 32; + return .{ .span = span, .launched = std.math.divCeil(i32, count, span) catch unreachable }; + } +}; + +/// Keep prefill and short selections direct; split long decode selections. +pub fn selectNumSegments(count: i32, decode: bool) i32 { + std.debug.assert(count > 0); + return if (decode) @min(8, std.math.divCeil(i32, count, 256) catch unreachable) else 1; +} + +pub const SparseAttentionConfig = struct { + dtype: metal.Type, + head_size: i32, + num_heads: i32, + num_queries_per_kv: i32, + block_size: i32, + indices_count: i32, + max_blocks: i32, + num_seqs: i32, + use_sinks: bool, + num_segments: i32 = 1, +}; + +pub const SparseAttention = Attention(false); +pub const SparseAttentionSplit = Attention(true); + +fn Attention(comptime split_k: bool) type { + return struct { + pub const Config = SparseAttentionConfig; + + pub const Kernel = metal.Kernel(Config, .{ + .name = if (split_k) "sparse_unified_attention_split" else "sparse_unified_attention", + .inputs = &.{ "q", "k", "v", "indices", "block_tables", "seq_lens", "query_starts", "sinks" }, + .outputs = &.{"out"}, + .constants = &.{"scale"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) metal.FinishError!void { + const input: metal.ArgSpec = .{ .buffer = .{ .type = cfg.dtype, .const_ = true } }; + const int_input: metal.ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .scale = .{ .constant = .float }, + .q = input, + .k = input, + .v = input, + .indices = int_input, + .block_tables = int_input, + .seq_lens = int_input, + .query_starts = int_input, + .sinks = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = if (split_k) .float else cfg.dtype } }, + .group = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const query = a.group.x().to(.int); + const head = a.group.y().to(.int); + const lane = a.lane.to(.int); + std.debug.assert(split_k or cfg.num_segments == 1); + const partition = Segments.init(cfg.indices_count, cfg.num_segments); + const slot = a.group.z().to(.int); + // Launch the partial tail first, without launching unused capacity. + const segment = if (split_k) b.ternary(slot.eq(0), partition.launched - 1, slot.sub(1)) else b.constant(.int, 0); + const row_idx = query.to(.long).mul(cfg.num_heads).add(head); + const row = row_idx.mul(cfg.head_size); + // Split partials are FP32: [query, head, segment, head_size + 2]. + // The last two elements hold the running maximum and exponential sum. + const partial_row = row_idx.mul(cfg.num_segments).add(segment).mul(cfg.head_size + 2); + const chunks = @divTrunc(cfg.head_size + 31, 32); + const acc = b.zeroedArray(.float, @intCast(chunks)); + + // Padded query capacity is not part of any sequence. + var padding = b.openIf(query.ge(a.query_starts.load(cfg.num_seqs))); + { + if (!split_k) { + var dims = b.openFor(lane, cfg.head_size, 32); + { + a.out.store(row.add(dims.iv), b.constant(cfg.dtype, 0)); + dims.close(); + } + } + b.return_(); + padding.close(); + } + + const left = b.local(.int, 0); + const right = b.local(.int, cfg.num_seqs); + var search = b.openLoop(.{}); + search.body(left.lt(right)); + { + const mid = left.add(right).div(2); + var half = b.openIfElse(a.query_starts.load(mid).le(query), .{}); + { + left.assign(mid.add(1)); + half.elseBranch(); + } + { + right.assign(mid); + half.close(); + } + search.close(); + } + const seq = left.sub(1); + const seq_len = a.seq_lens.load(seq); + const kv_head = head.div(cfg.num_queries_per_kv); + const kv_heads = @divExact(cfg.num_heads, cfg.num_queries_per_kv); + const maximum = b.local(.float, empty_max); + const sum = b.local(.float, 0.0); + if (cfg.use_sinks) { + // A sink contributes denominator mass once, in logical segment 0. + var sink = b.openIf(segment.eq(0)); + maximum.assign(a.sinks.load(head)); + sum.assign(1.0); + sink.close(); + } + const q_reg = b.zeroedArray(.float, @intCast(chunks)); + var load_q = b.openFor(0, chunks, 1); + { + const dim = lane.add(load_q.iv.mul(32)); + var valid = b.openIf(dim.lt(cfg.head_size)); + { + q_reg.store(load_q.iv, a.q.load(row.add(dim)).to(.float)); + valid.close(); + } + load_q.close(); + } + const start = if (split_k) segment.mul(partition.span) else b.constant(.int, 0); + const end = if (split_k) b.min(start.add(partition.span), cfg.indices_count) else b.constant(.int, cfg.indices_count); + var slots = b.openFor(start, end, 1); + { + const position = a.indices.load(query.to(.long).mul(cfg.indices_count).add(slots.iv)); + // The branch is uniform across the SIMD group. Invalid positions + // never access the page table or caches (which may contain NaNs). + var valid = b.openIf(position.ge(0).bitAnd(position.lt(seq_len))); + { + const page = a.block_tables.load(seq.to(.long).mul(cfg.max_blocks).add(position.div(cfg.block_size))).to(.long); + const offset = page.mul(cfg.block_size).add(position.rem(cfg.block_size)).mul(kv_heads).add(kv_head).mul(cfg.head_size); + const dot = b.local(.float, 0.0); + var dims = b.openFor(0, chunks, 1); + { + const dim = lane.add(dims.iv.mul(32)); + var present = b.openIf(dim.lt(cfg.head_size)); + { + dot.assign(dot.add(q_reg.load(dims.iv).mul(a.k.load(offset.add(dim)).to(.float)))); + present.close(); + } + dims.close(); + } + const score = b.simdSum(dot).mul(a.scale); + const next_max = b.max(maximum.rvalue(), score); + const alpha = b.ternary(sum.eq(0.0), 0.0, b.exp(maximum.sub(next_max))); + const weight = b.exp(score.sub(next_max)); + var update = b.openFor(0, chunks, 1); + { + const dim = lane.add(update.iv.mul(32)); + var present = b.openIf(dim.lt(cfg.head_size)); + { + acc.store(update.iv, acc.load(update.iv).mul(alpha).add(weight.mul(a.v.load(offset.add(dim)).to(.float)))); + present.close(); + } + update.close(); + } + sum.assign(sum.mul(alpha).add(weight)); + maximum.assign(next_max); + valid.close(); + } + slots.close(); + } + if (split_k) { + var stats = b.openIf(lane.eq(0)); + a.out.store(partial_row.add(cfg.head_size), maximum); + a.out.store(partial_row.add(cfg.head_size + 1), sum); + stats.close(); + } + const denominator = b.ternary(sum.gt(0.0), sum, 1.0); + var store = b.openFor(0, chunks, 1); + { + const dim = lane.add(store.iv.mul(32)); + var present = b.openIf(dim.lt(cfg.head_size)); + { + if (split_k) { + a.out.store(partial_row.add(dim), acc.load(store.iv)); + } else { + a.out.store(row.add(dim), acc.load(store.iv).div(denominator).to(cfg.dtype)); + } + present.close(); + } + store.close(); + } + } + }; +} + +/// Merge only launched, nonempty partials using their softmax statistics. +pub const SparseReduce = struct { + pub const Config = struct { + dtype: metal.Type, + head_size: i32, + num_heads: i32, + num_segments: i32, + launched_segments: i32, + }; + pub const Kernel = metal.Kernel(Config, .{ + .name = "sparse_attention_reduce", + .inputs = &.{ "partials", "query_starts" }, + .outputs = &.{"out"}, + .constants = &.{"num_seqs"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) metal.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .buffer = .{ .type = .float, .const_ = true } }, + .query_starts = .{ .buffer = .{ .type = .int, .const_ = true } }, + .out = .{ .buffer = .{ .type = cfg.dtype } }, + .num_seqs = .{ .constant = .int }, + .group = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const query = a.group.x().to(.int); + const row = query.to(.long).mul(cfg.num_heads).add(a.group.y()); + const lane = a.lane.to(.int); + var padding = b.openIf(query.ge(a.query_starts.load(a.num_seqs))); + { + var dims = b.openFor(lane, cfg.head_size, 32); + a.out.store(row.mul(cfg.head_size).add(dims.iv), b.constant(cfg.dtype, 0)); + dims.close(); + b.return_(); + padding.close(); + } + const chunks = @divTrunc(cfg.head_size + 31, 32); + const acc = b.zeroedArray(.float, @intCast(chunks)); + const maximum = b.local(.float, empty_max); + const sum = b.local(.float, 0.0); + var segments = b.openFor(0, cfg.launched_segments, 1); + { + const base = row.mul(cfg.num_segments).add(segments.iv).mul(cfg.head_size + 2); + const mass = a.partials.load(base.add(cfg.head_size + 1)); + // Empty segments and unused scratch capacity must not affect the + // maximum or read partial numerators (which may be uninitialized). + var nonempty = b.openIf(mass.gt(0.0)); + { + const m = a.partials.load(base.add(cfg.head_size)); + const next_max = b.max(maximum.rvalue(), m); + const alpha = b.ternary(sum.eq(0.0), 0.0, b.exp(maximum.sub(next_max))); + const beta = b.exp(m.sub(next_max)); + var dims = b.openFor(0, chunks, 1); + { + const dim = lane.add(dims.iv.mul(32)); + var valid = b.openIf(dim.lt(cfg.head_size)); + acc.store(dims.iv, acc.load(dims.iv).mul(alpha).add(a.partials.load(base.add(dim)).mul(beta))); + valid.close(); + dims.close(); + } + sum.assign(sum.mul(alpha).add(mass.mul(beta))); + maximum.assign(next_max); + nonempty.close(); + } + segments.close(); + } + const denominator = b.ternary(sum.gt(0.0), sum, 1.0); + var dims = b.openFor(0, chunks, 1); + { + const dim = lane.add(dims.iv.mul(32)); + var valid = b.openIf(dim.lt(cfg.head_size)); + a.out.store(row.mul(cfg.head_size).add(dim), acc.load(dims.iv).div(denominator).to(cfg.dtype)); + valid.close(); + dims.close(); + } + } +}; + +/// Inputs and output use the canonical contiguous layouts of the Metal backend. +pub fn call(inputs: SparseAttention.Kernel.Inputs, output: zml.Shape, cfg: SparseAttentionConfig, scale: f32) zml.Tensor { + const grid: [3]u32 = .{ @intCast(output.dim(0)), @intCast(cfg.num_heads), 1 }; + if (cfg.num_segments == 1) return SparseAttention.Kernel.call(inputs, .{ .out = output }, .{ + .cfg = cfg, + .constants = .{ .scale = .of(scale) }, + .grid = grid, + .threadgroup = .{ 32, 1, 1 }, + }).out; + const partition = Segments.init(cfg.indices_count, cfg.num_segments); + const partials = SparseAttentionSplit.Kernel.call(inputs, .{ + .out = zml.Shape.init(.{ output.dim(0), cfg.num_heads, cfg.num_segments, cfg.head_size + 2 }, .f32), + }, .{ + .cfg = cfg, + .constants = .{ .scale = .of(scale) }, + .grid = .{ grid[0], grid[1], @intCast(partition.launched) }, + .threadgroup = .{ 32, 1, 1 }, + }).out; + return SparseReduce.Kernel.call(.{ .partials = partials, .query_starts = inputs.query_starts }, .{ .out = output }, .{ + .cfg = .{ + .dtype = cfg.dtype, + .head_size = cfg.head_size, + .num_heads = cfg.num_heads, + .num_segments = cfg.num_segments, + .launched_segments = partition.launched, + }, + .constants = .{ .num_seqs = .of(cfg.num_seqs) }, + .grid = grid, + .threadgroup = .{ 32, 1, 1 }, + }).out; +} diff --git a/zml/kernels/paged_attention/metal/sparse_unified_attention_test.zig b/zml/kernels/paged_attention/metal/sparse_unified_attention_test.zig new file mode 100644 index 00000000..2f3fb8f6 --- /dev/null +++ b/zml/kernels/paged_attention/metal/sparse_unified_attention_test.zig @@ -0,0 +1,237 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const sparse = @import("sparse_unified_attention.zig"); +const attention = @import("metal.zig"); + +test "Metal sparse attention emits verified modules" { + for ([_]zml.kernel.metal.Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 1, 33, 64, 128, 256, 512 }) |head_size| { + for ([_]bool{ false, true }) |sinks| { + inline for (.{ sparse.SparseAttention, sparse.SparseAttentionSplit }) |Producer| { + const source = try Producer.Kernel.emit(std.testing.allocator, .{ + .dtype = dtype, + .head_size = head_size, + .num_heads = 4, + .num_queries_per_kv = 2, + .block_size = 4, + .indices_count = 7, + .max_blocks = 2, + .num_seqs = 4, + .use_sinks = sinks, + .num_segments = if (Producer == sparse.SparseAttentionSplit) 16 else 1, + }); + std.testing.allocator.free(source); + } + const reduce = try sparse.SparseReduce.Kernel.emit(std.testing.allocator, .{ + .dtype = dtype, + .head_size = head_size, + .num_heads = 4, + .num_segments = 16, + .launched_segments = 2, + }); + std.testing.allocator.free(reduce); + } + } + } +} + +test "Metal sparse attention partitions selection slots" { + try std.testing.expectEqual(sparse.Segments{ .span = 32, .launched = 2 }, sparse.Segments.init(33, 16)); + try std.testing.expectEqual(sparse.Segments{ .span = 192, .launched = 3 }, sparse.Segments.init(513, 3)); + try std.testing.expectEqual(@as(i32, 1), sparse.selectNumSegments(513, false)); + try std.testing.expectEqual(@as(i32, 1), sparse.selectNumSegments(256, true)); + try std.testing.expectEqual(@as(i32, 3), sparse.selectNumSegments(513, true)); + try std.testing.expectEqual(@as(i32, 8), sparse.selectNumSegments(8192, true)); +} + +const Case = struct { + dim: usize, + dtype: zml.DataType, + sinks: bool, + empty: bool = false, + decode: bool = false, + count: usize = 7, + segments: ?i32 = null, + holes: bool = false, +}; +const Inputs = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + indices: zml.Tensor, + table: zml.Tensor, + lengths: zml.Tensor, + starts: zml.Tensor, + sinks: zml.Tensor, + + fn init(c: Case) Inputs { + return .{ + .q = .init(.{ .b = 5, .hkv = 2, .hg = 2, .hd = c.dim }, .f32), + .k = .init(.{ .page = 4, .k_chunk = 4, .hkv = 2, .hd = c.dim }, .f32), + .v = .init(.{ .page = 4, .k_chunk = 4, .hkv = 2, .hd = c.dim }, .f32), + .indices = .init(.{ .b = 5, .topk = c.count }, .i32), + .table = .init(.{ .b = 4, .p = 2 }, .i32), + .lengths = .init(.{ .b = 4 }, .i32), + .starts = .init(.{ .b = 5 }, .i32), + .sinks = .init(.{4}, .f32), + }; + } +}; + +fn forward(i: Inputs, c: Case) zml.Tensor { + if (c.segments) |segments| { + const q = i.q.convert(c.dtype).merge(.{ .h = .{ .hkv, .hg } }); + return sparse.call(.{ + .q = q, + .k = i.k.convert(c.dtype), + .v = i.v.convert(c.dtype), + .indices = i.indices, + .block_tables = i.table, + .seq_lens = i.lengths, + .query_starts = i.starts, + .sinks = i.sinks, + }, q.shape(), .{ + .dtype = zml.kernel.metal.from(c.dtype), + .head_size = @intCast(c.dim), + .num_heads = 4, + .num_queries_per_kv = 2, + .block_size = 4, + .indices_count = @intCast(c.count), + .max_blocks = 2, + .num_seqs = 4, + .use_sinks = c.sinks, + .num_segments = segments, + }, 0.25).splitAxis(.h, .{ .hkv = 2, .hg = 2 }).convert(.f32); + } + return attention.pagedAttention(.{ + .block_table = i.table, + .seq_lens = i.lengths, + .query_start_len = i.starts, + .options_ = .{ .batch_size = 4, .max_num_pages = 2, .max_seqlen_q = if (c.decode) 1 else 2, .is_prefill = !c.decode }, + }, i.q.convert(c.dtype), i.k.convert(c.dtype), i.v.convert(c.dtype), .{ + .mask = .{ .indices = i.indices }, + .sink = if (c.sinks) i.sinks else null, + .scale = 0.25, + }).convert(.f32); +} + +test "Metal sparse attention matches CPU reference" { + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + for ([_]Case{ + .{ .dim = 33, .dtype = .f16, .sinks = false }, + .{ .dim = 128, .dtype = .bf16, .sinks = true }, + .{ .dim = 512, .dtype = .bf16, .sinks = true, .decode = true, .count = 513 }, + .{ .dim = 33, .dtype = .f16, .sinks = false, .decode = true, .count = 257 }, + .{ .dim = 64, .dtype = .bf16, .sinks = true, .decode = true, .count = 513, .segments = 8, .holes = true }, + .{ .dim = 33, .dtype = .f16, .sinks = false, .decode = true, .count = 33, .segments = 16 }, + .{ .dim = 64, .dtype = .bf16, .sinks = false, .decode = true, .count = 513, .empty = true }, + .{ .dim = 64, .dtype = .f16, .sinks = true, .decode = true, .count = 513, .empty = true }, + .{ .dim = 64, .dtype = .bf16, .sinks = false, .empty = true }, + .{ .dim = 64, .dtype = .f16, .sinks = true, .empty = true }, + }) |c| try check(c, platform); +} + +fn check(c: Case, platform: *const zml.Platform) !void { + const a = std.testing.allocator; + const io = std.testing.io; + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + const alloc = arena.allocator(); + const spec = Inputs.init(c); + const q = try alloc.alloc(f32, spec.q.count()); + const k = try alloc.alloc(f32, spec.k.count()); + const v = try alloc.alloc(f32, spec.v.count()); + const indices = try alloc.alloc(i32, spec.indices.count()); + for (q, 0..) |*x, j| x.* = @as(f32, @floatFromInt(@as(i32, @intCast(j % 11)) - 5)) / 8; + for (k, 0..) |*x, j| x.* = @as(f32, @floatFromInt(@as(i32, @intCast(j % 13)) - 6)) / 8; + for (v, 0..) |*x, j| x.* = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 8; + // Empty sequences, a repeated query start, prefill, and padded capacity. + const starts: [5]i32 = if (c.decode) .{ 0, 1, 1, 2, 3 } else .{ 0, 2, 2, 3, 4 }; + const lengths = [_]i32{ 5, 0, 3, 0 }; + const table = [_]i32{ 2, 0, -1, -1, 1, -1, -1, -1 }; + const sinks = [_]f32{ -2, 0, 1, 3 }; + const pattern = [_]i32{ 4, -1, 0, 2, 0, 5, -7 }; + for (indices, 0..) |*index, j| index.* = pattern[j % c.count % pattern.len]; + if (c.holes) { + @memset(indices, -1); + for (0..5) |t| { + indices[t * c.count + 200] = 2; + indices[(t + 1) * c.count - 1] = 0; + } + } + if (c.empty) { + @memset(indices, -1); + @memset(k, std.math.nan(f32)); + @memset(v, std.math.nan(f32)); + } + var buffers: zml.Bufferized(Inputs) = undefined; + var initialized: usize = 0; + defer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, j| { + if (j < initialized) @field(buffers, name).deinit(); + }; + const host = .{ .q = q, .k = k, .v = v, .indices = indices, .table = &table, .lengths = &lengths, .starts = &starts, .sinks = &sinks }; + inline for (@typeInfo(Inputs).@"struct".field_names) |name| { + @field(buffers, name) = try zml.Buffer.fromBytes(io, platform, @field(spec, name).shape(), std.mem.sliceAsBytes(@field(host, name))); + initialized += 1; + } + const exe = try platform.compileFn(a, io, forward, .{ spec, c }, .{}); + defer exe.deinit(); + var runner = try exe.runner(a); + defer runner.deinit(a); + var output: zml.Buffer = undefined; + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + var result = try output.toSliceAlloc(a, io); + defer result.free(a); + const actual = result.items(f32); + if (c.decode and (c.segments orelse sparse.selectNumSegments(@intCast(c.count), true)) > 1) { + var direct_case = c; + direct_case.segments = 1; + const direct_exe = try platform.compileFn(a, io, forward, .{ spec, direct_case }, .{}); + defer direct_exe.deinit(); + var direct_runner = try direct_exe.runner(a); + defer direct_runner.deinit(a); + var direct_output: zml.Buffer = undefined; + direct_runner.run(io, .{buffers}, .{&direct_output}, .{ .wait = true }); + defer direct_output.deinit(); + try zml.testing.expectClose(io, output, direct_output, .{ .absolute_tolerance = 0.004, .relative_tolerance = 0.004, .minimum_close_fraction = 1 }); + } + const scores = try alloc.alloc(f32, c.count); + for (0..5) |t| { + const active = t < starts[4]; + var seq: usize = 0; + while (seq < 3 and starts[seq + 1] <= t) : (seq += 1) {} + for (0..4) |h| { + var maximum: f32 = if (c.sinks) sinks[h] else -std.math.inf(f32); + for (indices[t * c.count ..][0..c.count], 0..) |pos, slot| { + scores[slot] = -std.math.inf(f32); + if (!active or pos < 0 or pos >= lengths[seq]) continue; + const p: usize = @intCast(pos); + const page: usize = @intCast(table[seq * 2 + p / 4]); + const off = ((page * 4 + p % 4) * 2 + h / 2) * c.dim; + var dot: f32 = 0; + for (0..c.dim) |d| dot += q[(t * 4 + h) * c.dim + d] * k[off + d]; + scores[slot] = dot * 0.25; + maximum = @max(maximum, scores[slot]); + } + if (!std.math.isFinite(maximum)) maximum = 0; + var sum: f32 = if (c.sinks and active) @exp(sinks[h] - maximum) else 0; + for (scores) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..c.dim) |d| { + var numerator: f32 = 0; + for (indices[t * c.count ..][0..c.count], scores) |pos, weight| { + if (weight == 0) continue; + const p: usize = @intCast(pos); + const page: usize = @intCast(table[seq * 2 + p / 4]); + numerator += weight * v[((page * 4 + p % 4) * 2 + h / 2) * c.dim + d]; + } + const expected = numerator / (if (sum > 0) sum else 1); + try std.testing.expectApproxEqAbs(expected, actual[(t * 4 + h) * c.dim + d], @as(f32, if (c.dtype == .bf16) 0.004 else 0.0005)); + } + } + } +}