From 65b2a176dd7d076f3ff6b4424be61e62c966a8f0 Mon Sep 17 00:00:00 2001 From: dmehala Date: Sun, 15 Nov 2026 01:30:54 +0000 Subject: [PATCH 1/3] wip --- zml/kernels/paged_attention/cute/BUILD.bazel | 12 + .../paged_attention/cute/benchmark.zig | 194 ++++-- zml/kernels/paged_attention/cute/cases.zig | 69 +- .../paged_attention/cute/correctness_test.zig | 67 +- zml/kernels/paged_attention/cute/cute.zig | 83 ++- zml/kernels/paged_attention/cute/fixtures.zig | 84 ++- .../paged_attention/cute/reference.zig | 70 +- .../paged_attention/cute/sparse_attention.zig | 615 ++++++++++++++++++ .../paged_attention/paged_attention.zig | 2 +- 9 files changed, 1077 insertions(+), 119 deletions(-) create mode 100644 zml/kernels/paged_attention/cute/sparse_attention.zig diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index 6329ae12..39d3fbbf 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -7,11 +7,22 @@ filegroup( srcs = [ "attention.zig", "cute.zig", + "sparse_attention.zig", "//zml/kernels/paged_attention/cute/sm10x", "//zml/kernels/paged_attention/cute/sm120", ], ) +zig_test( + name = "reference_test", + srcs = [ + "cases.zig", + "fixtures.zig", + ], + main = "reference.zig", + deps = ["//zml"], +) + zig_test( name = "correctness_test", srcs = [ @@ -36,6 +47,7 @@ zig_binary( srcs = [ "cases.zig", "fixtures.zig", + "reference.zig", ], main = "benchmark.zig", tags = ["manual"], diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 587af69f..dafd2c81 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -2,130 +2,188 @@ const std = @import("std"); const zml = @import("zml"); const cases = @import("cases.zig"); const fixtures = @import("fixtures.zig"); +const reference = @import("reference.zig"); const Inputs = fixtures.Inputs; +const Args = struct { + profile_dir: ?[]const u8 = null, + sparse: bool = false, + stress: bool = false, + all_backends: bool = false, + backend: ?fixtures.Backend = null, + case: ?[]const u8 = null, + dtype: zml.DataType = .bf16, + warmup: usize = 10, + samples: usize = 101, +}; + pub fn main(init: std.process.Init) !void { + const args = try parseArgs(init); const platform = try zml.Platform.auto(init.gpa, init.io, .{}); defer platform.deinit(init.gpa, init.io); - if (platform.target != .cuda) return error.CudaRequired; const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; - std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ - cc.major, - cc.minor, - platform.fmtVerbose(), + std.debug.print("PAGED_ATTENTION dtype={s} page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + @tagName(args.dtype), cc.major, cc.minor, platform.fmtVerbose(), }); + std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload, reference validation and output destruction)\n", .{platform.devices.len}); - std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); + const suite: []const cases.Case = if (args.stress) &cases.sparseStressBenchmarks else if (args.sparse) &cases.sparseBenchmarks else &cases.benchmarks; var failures: usize = 0; - for (cases.benchmarks) |c| { - std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ - c.name, - c.head_dim, - c.batch, - c.kv_heads, - c.group, - c.prefill, - if (c.prefill > 0) c.chunk else 1, - c.kv_len, - @tagName(c.mask), - }); - - runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { - failures += 1; - std.debug.print("FAIL error={s}\n", .{@errorName(err)}); - }; + var selected: usize = 0; + for (suite, 0..) |c, index| { + if (args.case) |filter| { + const wanted_index: ?usize = std.fmt.parseInt(usize, filter, 10) catch null; + if (wanted_index) |wanted| { + if (wanted != index) continue; + } else if (!std.mem.eql(u8, filter, c.name)) continue; + } + selected += 1; + const backends = std.enums.values(fixtures.Backend); + for (0..backends.len) |rank| { + const backend = backends[(rank + index) % backends.len]; + if (args.backend) |wanted| { + if (wanted != backend) continue; + } else if (!args.sparse and !args.all_backends and backend != .cute) continue; + // Triton's GB300 dot accumulator has a documented 16K-element + // tile limit. Keep optional stress runs from faulting the device. + if (backend == .triton and c.mask == .indices and cc.major == 10 and cc.minor == 3 and c.group * c.head_dim > 16 * 1024) { + std.debug.print("SKIP index={d} name={s} backend=triton reason=gb300_triton_head_tile_limit\n", .{ index, c.name }); + continue; + } + std.debug.print("CASE index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} mask={s} backend={s}\n", .{ + index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(c.mask), @tagName(backend), + }); + runBenchmark(init.gpa, init.io, platform, mesh, c, index, backend, args) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } } - + if (selected == 0) return error.UnknownCase; if (failures != 0) return error.BenchmarkFailed; } -fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { - // Keep number of head per GPU constant. +fn parseArgs(init: std.process.Init) !Args { + var result: Args = .{}; + var it = init.minimal.args.iterate(); + _ = it.next(); + while (it.next()) |arg| { + if (std.mem.eql(u8, arg, "--sparse")) { + result.sparse = true; + } else if (std.mem.eql(u8, arg, "--stress")) { + result.sparse = true; + result.stress = true; + } else if (std.mem.startsWith(u8, arg, "--backend=")) { + const value = arg["--backend=".len..]; + result.all_backends = std.mem.eql(u8, value, "all"); + result.backend = if (std.mem.eql(u8, value, "all")) null else std.meta.stringToEnum(fixtures.Backend, value) orelse return error.InvalidBackend; + } else if (std.mem.startsWith(u8, arg, "--case=")) { + result.case = arg["--case=".len..]; + } else if (std.mem.startsWith(u8, arg, "--dtype=")) { + result.dtype = std.meta.stringToEnum(zml.DataType, arg["--dtype=".len..]) orelse return error.InvalidDType; + if (result.dtype != .bf16 and result.dtype != .f16) return error.InvalidDType; + } else if (std.mem.startsWith(u8, arg, "--profile-dir=")) { + result.profile_dir = arg["--profile-dir=".len..]; + } else if (std.mem.startsWith(u8, arg, "--warmup=")) { + result.warmup = try std.fmt.parseInt(usize, arg["--warmup=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--samples=")) { + result.samples = try std.fmt.parseInt(usize, arg["--samples=".len..], 10); + if (result.samples == 0) return error.InvalidSamples; + } else if (std.mem.eql(u8, arg, "--help")) { + std.debug.print("paged attention benchmark [--sparse|--stress] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); + std.process.exit(0); + } else return error.UnknownArgument; + } + return result; +} + +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case, index: usize, backend: fixtures.Backend, args: Args) !void { + // Keep the number of KV heads per GPU constant. var global_case = c; global_case.kv_heads *= platform.devices.len; - var inputs = Inputs.init(global_case); + var inputs = Inputs.initDType(global_case, args.dtype); inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); defer exe.deinit(); const compile_us = elapsedUs(compile_start, io); - var runner = try exe.runner(a); defer runner.deinit(a); - var arena = std.heap.ArenaAllocator.init(a); defer arena.deinit(); - const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); - - var buffers = try fixtures.upload(io, platform, inputs, fixture); + if (args.dtype == .f16) fixture.quantizeF16(); + var buffers = try fixtures.upload(a, io, platform, inputs, fixture); defer zml.Buffer.deinitAll(Inputs, &buffers); var output: zml.Buffer = undefined; - - // Warmup - for (0..5) |_| { + const max_error = validate: { + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + break :validate if (args.dtype == .f16) + try reference.check(a, global_case, fixture, host.items(f16)) + else + try reference.check(a, global_case, fixture, host.items(zml.floats.BFloat16)); + }; + for (0..args.warmup) |_| { runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); output.deinit(); } - - var samples: [101]f64 = undefined; - var sample_count: usize = 0; - var accumulated_us: f64 = 0; - while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { + const session = try std.fmt.allocPrint(a, "{s}-{d}-{s}", .{ @tagName(args.dtype), index, @tagName(backend) }); + defer a.free(session); + var profiler = if (args.profile_dir) |path| try platform.profiler(a, io, .{ + .repository_path = path, + .session_id = session, + .host_tracer_level = 0, + .device_tracer_level = 1, + .python_tracer_level = 0, + .raise_error_on_start_failure = true, + }) else null; + defer if (profiler) |*p| p.deinit(); + if (profiler) |*p| try p.start(); + const samples = try a.alloc(f64, args.samples); + defer a.free(samples); + for (samples) |*sample| { const call_start: std.Io.Timestamp = .now(io, .awake); runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - samples[sample_count] = elapsedUs(call_start, io); - accumulated_us += samples[sample_count]; - sample_count += 1; + sample.* = elapsedUs(call_start, io); output.deinit(); } - - std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); - const median = if (sample_count % 2 == 0) - (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 + if (profiler) |*p| _ = try p.stop(); + std.mem.sort(f64, samples, {}, std.sort.asc(f64)); + const median = if (samples.len % 2 == 0) + (samples[samples.len / 2 - 1] + samples[samples.len / 2]) / 2 else - samples[sample_count / 2]; - - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ - c.name, - c.head_dim, - c.batch, - c.kv_heads, - c.group, - c.prefill, - if (c.prefill > 0) c.chunk else 1, - c.kv_len, - compile_us / 1000, - sample_count, - median, - samples[sample_count / 10], - samples[(sample_count - 1) * 9 / 10], + samples[samples.len / 2]; + std.debug.print("RESULT index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_outputs max_abs={d:.6} compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ + index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(args.dtype), max_error, compile_us / 1000, samples.len, median, samples[samples.len / 10], samples[(samples.len - 1) * 9 / 10], }); } -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtures.Backend) zml.Tensor { const Call = struct { inputs: Inputs, case: cases.Case, + backend: fixtures.Backend, fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forward(self.inputs, self.case); + return fixtures.forwardBackend(self.inputs, self.case, self.backend); } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c, .backend = backend }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index e8dedeaa..0c2f4c04 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -15,16 +15,19 @@ pub const Case = struct { kv_len: usize = 65, kv_capacity: ?usize = null, inactive: usize = 0, + token_capacity: ?usize = null, shuffle_pages: bool = false, skew: bool = false, mask: Mask = .causal, + topk: usize = 8, empty_indices: bool = false, window: u32 = 17, sink: bool = false, + sink_value: ?f32 = null, scale: ?f32 = null, pub fn tokens(c: Case) usize { - return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; + return c.token_capacity orelse (c.prefill * c.chunk + c.batch - c.prefill - c.inactive); } pub fn pages(c: Case) usize { return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; @@ -101,3 +104,67 @@ pub const benchmarks = blk: { } break :blk result; }; + +// Selection-slot tails, GQA head tails, and logical-to-physical page gathers +// vary independently. Every nonempty fixture includes duplicate/invalid slots. +pub const sparseCorrectness = [_]Case{ + .{ .name = "sparse_tiny_mha", .head_dim = 128, .group = 1, .mask = .indices, .topk = 8, .shuffle_pages = true }, + .{ .name = "sparse_d32_mha", .head_dim = 32, .group = 1, .mask = .indices, .topk = 32 }, + .{ .name = "sparse_d64_mqa", .head_dim = 64, .kv_heads = 1, .group = 8, .mask = .indices, .topk = 65 }, + .{ .name = "sparse_d128_mixed", .head_dim = 128, .prefill = 1, .chunk = 9, .mask = .indices, .topk = 65, .shuffle_pages = true }, + .{ .name = "sparse_d256_mixed_sink", .head_dim = 256, .prefill = 1, .chunk = 3, .mask = .indices, .topk = 32, .sink = true }, + .{ .name = "sparse_d512", .head_dim = 512, .mask = .indices, .topk = 65, .shuffle_pages = true }, + .{ .name = "sparse_group5", .head_dim = 64, .group = 5, .prefill = 1, .chunk = 5, .mask = .indices, .topk = 65 }, + .{ .name = "sparse_group19", .head_dim = 128, .group = 19, .mask = .indices, .topk = 65, .sink = true }, + .{ .name = "sparse_d512_group19", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 19, .mask = .indices, .topk = 32 }, + .{ .name = "sparse_long_selection", .head_dim = 128, .batch = 1, .kv_len = 2049, .mask = .indices, .topk = 512, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_long_tail", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .indices, .topk = 513, .shuffle_pages = true }, + .{ .name = "sparse_split_tail", .head_dim = 64, .batch = 1, .kv_len = 4097, .mask = .indices, .topk = 1025, .sink = true }, + .{ .name = "sparse_all_invalid", .head_dim = 128, .prefill = 1, .chunk = 3, .mask = .indices, .topk = 65, .empty_indices = true }, + .{ .name = "sparse_all_invalid_sink", .head_dim = 256, .mask = .indices, .topk = 512, .empty_indices = true, .sink = true }, + .{ .name = "sparse_skew_inactive", .head_dim = 128, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 5, .kv_len = 129, .mask = .indices, .topk = 65, .skew = true, .shuffle_pages = true }, + .{ .name = "sparse_spare_capacity", .head_dim = 128, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 16, .kv_len = 129, .mask = .indices, .topk = 65, .shuffle_pages = true }, + .{ .name = "sparse_negative_scale", .head_dim = 64, .mask = .indices, .topk = 32, .scale = -0.125 }, + .{ .name = "sparse_all_invalid_disabled_sink", .head_dim = 128, .mask = .indices, .topk = 8, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, + .{ .name = "sparse_zero_scale", .head_dim = 128, .mask = .indices, .topk = 65, .scale = 0, .sink = true }, +}; + +pub const sparseBenchmarks = blk: { + var result: [25]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]usize{ 1, 8 }) |batch| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ + .name = "sparse", + .head_dim = dim, + .batch = batch, + .prefill = if (mixed) @max(1, batch / 4) else 0, + .chunk = 16, + .kv_len = 8192, + .mask = .indices, + .topk = 512, + .shuffle_pages = true, + }; + i += 1; + } + } + } + result[16] = .{ .name = "sparse_short", .head_dim = 128, .batch = 8, .kv_len = 8192, .mask = .indices, .topk = 32, .shuffle_pages = true }; + result[17] = .{ .name = "sparse_odd_tail", .head_dim = 128, .batch = 8, .group = 8, .kv_len = 8192, .mask = .indices, .topk = 65, .shuffle_pages = true }; + result[18] = .{ .name = "sparse_large_gqa", .head_dim = 128, .batch = 1, .group = 19, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }; + result[19] = .{ .name = "sparse_mha", .head_dim = 128, .batch = 8, .group = 1, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; + result[20] = .{ .name = "sparse_batch64", .head_dim = 128, .batch = 64, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; + result[21] = .{ .name = "sparse_batch128", .head_dim = 128, .batch = 128, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; + result[22] = .{ .name = "sparse_tiny_mha", .head_dim = 128, .batch = 8, .group = 1, .kv_len = 8192, .mask = .indices, .topk = 8, .shuffle_pages = true }; + result[23] = .{ .name = "sparse_gqa32", .head_dim = 128, .batch = 8, .group = 32, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; + result[24] = .{ .name = "sparse_mqa16_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 16, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }; + break :blk result; +}; + +// These geometries expose backend resource limits; keep them separately +// selectable so one unsupported backend cannot interrupt the shared matrix. +pub const sparseStressBenchmarks = [_]Case{ + .{ .name = "sparse_mqa64_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 64, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }, + .{ .name = "sparse_mqa128_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 128, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }, +}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index 436b9d3a..deb4dbf8 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -27,7 +27,7 @@ test "paged attention correctness" { c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), }); - runCorrectness(a, io, platform, c, false) catch |err| { + runCorrectness(a, io, platform, c, .cute, .bf16) catch |err| { failures += 1; std.debug.print("FAIL error={s}\n", .{@errorName(err)}); }; @@ -44,19 +44,38 @@ test "FP16 paged attention" { .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, - }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); + }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, .cute, .f16); } -fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { - var inputs = i; - inputs.q = i.q.convert(.f16); - inputs.k = i.k.convert(.f16); - inputs.v = i.v.convert(.f16); - return fixtures.forward(inputs, c).convert(.bf16); +test "indexed paged attention BF16 all backends" { + try runSparseCorrectness(.bf16); } -fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { - const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); +test "indexed paged attention FP16 all backends" { + try runSparseCorrectness(.f16); +} + +fn runSparseCorrectness(dtype: zml.DataType) !void { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + var failures: usize = 0; + for (cases.sparseCorrectness) |c| { + for (std.enums.values(fixtures.Backend)) |backend| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(dtype), + }); + runCorrectness(std.testing.allocator, std.testing.io, platform, c, backend, dtype) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + } + try std.testing.expectEqual(@as(usize, 0), failures); +} + +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, backend: fixtures.Backend, dtype: zml.DataType) !void { + const inputs = Inputs.initDType(c, dtype); + const exe = try platform.compileFn(a, io, fixtures.forwardBackend, .{ inputs, c, backend }, .{ .program_name = "paged_attention_harness" }); defer exe.deinit(); var runner = try exe.runner(a); @@ -66,17 +85,9 @@ fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platfor defer arena.deinit(); const fixture = try fixtures.Fixture.init(arena.allocator(), c); - if (compute_f16) { - // Exactly representable in both formats, so the existing CPU reference - // receives precisely the values used by the FP16 tensor-core path. - for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { - for (values) |*v| if (std.math.isFinite(v.toF32())) { - v.* = .fromF32(@round(v.toF32() * 64) / 64); - }; - } - } + if (dtype == .f16) fixture.quantizeF16(); - var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); + var buffers = try fixtures.upload(a, io, platform, inputs, fixture); defer zml.Buffer.deinitAll(Inputs, &buffers); var output: zml.Buffer = undefined; @@ -86,8 +97,18 @@ fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platfor var host = try output.toSliceAlloc(a, io); defer host.free(a); - const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, + const max_error = if (dtype == .f16) + try reference.check(a, c, fixture, host.items(f16)) + else + try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); + if (backend != .triton) { + if (dtype == .f16) { + try reference.checkZeroTail(c, fixture, host.items(f16)); + } else { + try reference.checkZeroTail(c, fixture, host.items(zml.floats.BFloat16)); + } + } + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_active_outputs inactive_tail={s} max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(dtype), if (backend == .triton) "unspecified" else "zero_checked", max_error, }); } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ef78cc49..6d30693a 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -3,6 +3,7 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); pub const AttentionOptions = @import("../options.zig").AttentionOptions; const kernel = @import("attention.zig"); +const sparse = @import("sparse_attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); @@ -54,6 +55,7 @@ pub fn multiprocessorCount(platform: *const zml.Platform) usize { test { _ = kernel; + _ = sparse; _ = sm10x; _ = sm120; } @@ -64,20 +66,38 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { +fn validateInputs(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor) void { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); - if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) +} + +fn sparseIndices(q: zml.Tensor, indices: zml.Tensor) zml.Tensor { + stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); + stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices require .b and .topk tags", .{}); + stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); + return indices.transpose(.{ .b, .topk }); +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + validateInputs(q, k, v); + if (opts.mask == .indices) + return pagedSparseAttention(parameters, q, k, v, opts); + if (sm10x.isAvailable(zml.Compiler.current().platform)) return sm10x.pagedAttention(parameters, q, k, v, opts); - if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + if (sm120.isAvailable(zml.Compiler.current().platform)) return sm120.pagedAttention(parameters, q, k, v, opts); + return pagedAttentionGeneric(parameters, q, k, v, opts); +} + +pub fn pagedAttentionGeneric(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + validateInputs(q, k, v); const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); const indices = switch (opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), + .indices => |indices| sparseIndices(q_, indices), else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), }; const cfg: kernel.Config = .{ @@ -113,3 +133,58 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .block = .{ 32, 1, 1 }, }).out.transpose(q.shape()); } + +fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = sparseIndices(q_, opts.mask.indices); + // A short, single-head selection has no KV reuse to amortize MMA tiling. + if (q_.dim(.hg) == 1 and indices.dim(.topk) <= 8) + return pagedAttentionGeneric(parameters, q, k, v, opts); + var cfg: sparse.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .topk = indices.dim(.topk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .kv_tile = if (!parameters.options_.is_prefill and q_.dim(.hd) <= 256) 16 else if (q_.dim(.hd) == 512) 32 else 64, + .warps = if (!parameters.options_.is_prefill and q_.dim(.hd) <= 256) @max(1, @divTrunc(q_.dim(.hd), 128)) else 4, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const work = @max(1, cfg.tokens * cfg.kv_heads * cfg.headTiles()); + // Narrow decode CTAs keep about 64 FP32 output registers per thread. + // One selected tile per partition exposes parallelism at small batches. + const target = @as(i64, @intCast(multiprocessorCount(zml.Compiler.current().platform))) * 16; + const tiles = @divTrunc(cfg.topk + cfg.kv_tile - 1, cfg.kv_tile); + var desired: i64 = 1; + while (desired * work < target) desired *= 2; + cfg.splits = if (!parameters.options_.is_prefill) @max(1, @min(128, @min(tiles, desired))) else if (cfg.head_dim <= 128) 1 else @max(1, @min(32, @min(@divTrunc(@divTrunc(target, 8) + work - 1, work), @divTrunc(cfg.topk, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.scalar(0, .f32); + const out = sparse.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = if (cfg.splits > 1) zml.Shape.init(.{ cfg.splits, cfg.tokens, cfg.kv_heads * cfg.group, cfg.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.headTiles()), @intCast(cfg.splits) }, + .block = .{ @intCast(cfg.warps * 32), 1, 1 }, + }).out; + const result = if (cfg.splits > 1) sparse.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divTrunc(cfg.head_dim + 63, 64)) }, + .block = .{ 64, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index c0815d1d..435ec525 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -4,6 +4,8 @@ const zml = @import("zml"); const cases = @import("cases.zig"); +pub const Backend = enum { cute, cute_generic, triton }; + pub const Fixture = struct { q: []zml.floats.BFloat16, k: []zml.floats.BFloat16, @@ -24,14 +26,14 @@ pub const Fixture = struct { .lengths = try a.alloc(i32, c.batch), .starts = try a.alloc(i32, c.batch + 1), .sinks = try a.alloc(f32, c.heads()), - .indices = try a.alloc(i32, c.tokens() * cases.topk), + .indices = try a.alloc(i32, c.tokens() * c.topk), }; var rng = std.Random.DefaultPrng.init(42); const random = rng.random(); for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); - for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; + for (f.sinks, 0..) |*v, h| v.* = c.sink_value orelse (@as(f32, @floatFromInt(h % 5)) - 2); @memset(f.indices, -1); f.starts[0] = 0; for (0..c.batch) |s| { @@ -55,7 +57,15 @@ pub const Fixture = struct { } for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { const l: i32 = @intCast(len); - @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); + const boundary = [_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }; + for (f.indices[t * c.topk ..][0..c.topk], 0..) |*index, j| { + index.* = if (j < boundary.len) boundary[j] else switch (j % 17) { + 0 => -1, + 1 => l, + 2 => 0, + else => @intCast((j * 37 + t * 13) % len), + }; + } } } if (c.empty_indices) { @@ -63,9 +73,20 @@ pub const Fixture = struct { @memset(f.k, .fromF32(std.math.nan(f32))); @memset(f.v, .fromF32(std.math.nan(f32))); } + const active_tokens: usize = @intCast(f.starts[c.batch]); + @memset(f.q[active_tokens * c.heads() * c.head_dim ..], .fromF32(std.math.nan(f32))); return f; } + pub fn quantizeF16(f: Fixture) void { + // Both host reference and native FP16 uploads receive identical values. + for ([_][]zml.floats.BFloat16{ f.q, f.k, f.v }) |values| { + for (values) |*v| if (std.math.isFinite(v.toF32())) { + v.* = .fromF32(@round(v.toF32() * 64) / 64); + }; + } + } + pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; @@ -83,15 +104,19 @@ pub const Inputs = struct { indices: zml.Tensor, pub fn init(c: cases.Case) Inputs { + return initDType(c, .bf16); + } + + pub fn initDType(c: cases.Case, dtype: zml.DataType) Inputs { return .{ - .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), - .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), - .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, dtype), + .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, dtype), + .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, dtype), .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), .lengths = .init(.{ .b = c.batch }, .i32), .starts = .init(.{ .b = c.batch + 1 }, .i32), .sinks = .init(.{c.heads()}, .f32), - .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), + .indices = .init(.{ .b = c.tokens(), .topk = c.topk }, .i32), }; } @@ -111,30 +136,57 @@ pub const Inputs = struct { }; pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { - const parameters: zml.paged_attention.Parameters = .{ - .cute = .{ + return forwardBackend(i, c, .cute); +} + +pub fn forwardBackend(i: Inputs, c: cases.Case, backend: Backend) zml.Tensor { + const parameters: zml.paged_attention.cute.Parameters = .{ + .block_table = i.table, + .seq_lens = i.lengths, + .query_start_len = i.starts, + .options_ = .{ + .batch_size = c.batch, + .max_num_pages = c.pages(), + .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, + .is_prefill = c.prefill > 0, + }, + }; + return switch (backend) { + .cute => zml.paged_attention.cute.pagedAttention(parameters, i.q, i.k, i.v, i.options(c)), + .cute_generic => zml.paged_attention.cute.pagedAttentionGeneric(parameters, i.q, i.k, i.v, i.options(c)), + .triton => zml.paged_attention.triton.paged.pagedAttention(.{ .block_table = i.table, .seq_lens = i.lengths, .query_start_len = i.starts, .options_ = .{ - .batch_size = c.batch, - .max_num_pages = c.pages(), - .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, + .batch_size = @intCast(c.batch), + .max_num_pages = @intCast(c.pages()), + .max_seqlen_q = @intCast(if (c.prefill > 0) c.chunk else 1), .is_prefill = c.prefill > 0, }, - }, + }, i.q, i.k, i.v, i.options(c)), }; - return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); } -pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { +pub fn upload(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { var result: zml.Bufferized(Inputs) = undefined; var initialized: usize = 0; errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { if (index < initialized) @field(result, name).deinit(); }; inline for (@typeInfo(Inputs).@"struct".field_names) |name| { - @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + if (comptime std.mem.eql(u8, name, "q") or std.mem.eql(u8, name, "k") or std.mem.eql(u8, name, "v")) { + if (@field(inputs, name).dtype() == .f16) { + const values = try a.alloc(f16, @field(f, name).len); + defer a.free(values); + for (values, @field(f, name)) |*dst, src| dst.* = @floatCast(src.toF32()); + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(values)); + } else { + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + } + } else { + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + } initialized += 1; } return result; diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig index 1a030646..b80fabee 100644 --- a/zml/kernels/paged_attention/cute/reference.zig +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -10,7 +10,7 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u const start: usize = @intCast(f.starts[seq]); const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); const position = len - qlen + token - start; - const count = if (c.mask == .indices) cases.topk else len; + const count = if (c.mask == .indices) c.topk else len; const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); @@ -44,15 +44,19 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u } fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { - return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); + return if (c.mask == .indices) f.indices[token * c.topk + j] else @intCast(j); } -pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { +fn toF32(value: anytype) f32 { + return if (@TypeOf(value) == zml.floats.BFloat16) value.toF32() else @floatCast(value); +} + +pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: anytype) !f32 { const active_tokens: usize = @intCast(f.starts[c.batch]); for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { - if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; + if (!std.math.isFinite(toF32(v))) return error.NonFiniteOutput; } - const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); + const scores = try a.alloc(f32, @max(c.kv_len, c.topk)); defer a.free(scores); const expected = try a.alloc(f32, c.head_dim); defer a.free(expected); @@ -64,7 +68,7 @@ pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: [ row(c, f, s, t, h, scores, expected); const base = (t * c.heads() + h) * c.head_dim; for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { - const got = got_bf16.toF32(); + const got = toF32(got_bf16); const err = @abs(want - got); max_error = @max(max_error, err); // Fixed BF16 error budget; never adapted to observed failures. @@ -80,6 +84,13 @@ pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: [ return max_error; } +pub fn checkZeroTail(c: cases.Case, f: fixtures.Fixture, actual: anytype) !void { + const active_tokens: usize = @intCast(f.starts[c.batch]); + for (actual[active_tokens * c.heads() * c.head_dim ..]) |value| { + if (toF32(value) != 0) return error.NonzeroInactiveOutput; + } +} + test "reference single key, uniform causal scores, and sink" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -125,3 +136,50 @@ test "reference windows, noncausal, duplicate and invalid indices" { row(c, f, 0, 1, 0, &scores, &out); for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); } + +test "reference indexed selection tail, FP16 output and unused query capacity" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + const c: cases.Case = .{ + .head_dim = 64, + .batch = 1, + .kv_heads = 1, + .group = 1, + .prefill = 1, + .chunk = 2, + .kv_len = 2, + .mask = .indices, + .topk = 65, + .token_capacity = 4, + }; + const f = try fixtures.Fixture.init(a, c); + @memset(f.q[0 .. 2 * 64], .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + @memset(f.indices, -1); + f.indices[64] = 1; + f.indices[c.topk + 64] = 0; + const actual = try a.alloc(f16, c.tokens() * c.heads() * c.head_dim); + @memset(actual[0..64], 4); + @memset(actual[64..128], 2); + @memset(actual[128..], 0); + try std.testing.expectEqual(@as(f32, 0), try check(a, c, f, actual)); + try checkZeroTail(c, f, actual); + actual[128] = std.math.nan(f16); + try std.testing.expectError(error.NonzeroInactiveOutput, checkZeroTail(c, f, actual)); +} + +test "reference all invalid indexed slots with sink ignore poisoned cache" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .mask = .indices, .topk = 65, .empty_indices = true, .sink = true }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + var scores: [65]f32 = undefined; + var out: [64]f32 = undefined; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); + @memset(f.sinks, -std.math.inf(f32)); + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); +} diff --git a/zml/kernels/paged_attention/cute/sparse_attention.zig b/zml/kernels/paged_attention/cute/sparse_attention.zig new file mode 100644 index 00000000..3d15634f --- /dev/null +++ b/zml/kernels/paged_attention/cute/sparse_attention.zig @@ -0,0 +1,615 @@ +// Decode CTA width bounds the output fragment at 64 FP32 values per thread. +// Rows are query heads of one token sharing a KV head. +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + topk: i64, + dtype: cute.DType, + kv_tile: i64, + warps: i64 = 4, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + _ = c; + return 16; + } + pub fn headTiles(c: Config) i64 { + return @divTrunc(c.group + c.rows() - 1, c.rows()); + } + fn kvBase(c: Config) i64 { + return c.rows() * c.head_dim * 2; + } + fn pBase(c: Config) i64 { + return c.kvBase() + c.kv_tile * c.head_dim * 2; + } + fn redBase(c: Config) i64 { + return c.pBase() + c.rows() * c.kv_tile * 2; + } + fn indexBase(c: Config) i64 { + return c.redBase() + 2 * 64 * 4; + } + fn sharedBytes(c: Config) i64 { + return c.indexBase() + c.kv_tile * 8; + } +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_indexed_grouped", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { + var result: [n * 4]V = undefined; + for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); + return result; +} + +fn packAcc(b: *B, values: [4]V) V { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn loadI32(b: *B, ptr: V, index: V) V { + return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. +fn swizzle(row: V, col: V, width: i64) V { + return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); +} + +fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { + var result: [n]V = undefined; + for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + return result; +} + +fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { + const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ + (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); + return unpackStruct(b, n, result, .i32); +} + +fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +fn run(b: *B, c: Config) cute.FinishError!void { + switch (c.head_dim) { + inline 32, 64, 128, 256 => |d| switch (c.warps) { + inline 1, 2, 4 => |w| try runDimension(b, c, d, w), + else => unreachable, + }, + 512 => { + const d = 512; + std.debug.assert(c.warps == 4); + try runDimension(b, c, d, 4); + }, + else => unreachable, + } +} + +fn runDimension(b: *B, c: Config, comptime d: usize, comptime w: usize) cute.FinishError!void { + switch (c.kv_tile) { + 16 => if (w <= 2) switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTile(b, c, d, w, 16, dt), + else => unreachable, + } else unreachable, + inline 32, 64 => |n| switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), + else => unreachable, + }, + else => unreachable, + } +} + +const TileContext = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + indices: V, + sink: V, + out: V, + shared: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + token: V, + klen: V, + split: V, + + // Resolve each selected slot once, then share its physical page across + // every head, feature vector, and both K/V loads. Invalid logical indices + // never dereference the page table; cp.async zero-fills their K/V rows. + fn gather(t: TileContext, key_base: V) void { + const b = t.b; + const c = t.c; + var selected = b.openFor(t.tid, c.kv_tile, c.warps * 32, .{}); + { + const address = t.shared.add(c.indexBase()).add(selected.iv.mul(8)); + b.storeWords(1, address, .shared, .{b.constant(.i32, -1)}); + b.storeWords(1, address.add(4), .shared, .{b.constant(.i32, 0)}); + const slot = key_base.add(selected.iv); + var in_range = b.openIf(slot.lt(c.topk)); + { + const key = loadI32(b, t.indices, t.token.to(.i64).mul(c.topk).add(slot.to(.i64))); + var valid = b.openIf(key.ge(0).bitAnd(key.lt(t.klen))); + { + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(key.div(c.page_size))); + b.storeWords(1, address, .shared, .{page}); + b.storeWords(1, address.add(4), .shared, .{key.rem(c.page_size)}); + valid.close(); + } + in_range.close(); + } + selected.close(); + } + b.syncThreads(); + } + + fn validSlot(t: TileContext, row: V) V { + return t.b.loadWords(1, t.shared.add(t.c.indexBase()).add(row.mul(8)), .shared, .{})[0].ge(0); + } + + fn loadKV(t: TileContext, source: V) void { + const b = t.b; + const c = t.c; + var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), c.warps * 32, .{}); + { + const row = copy.iv.div(@divExact(c.head_dim, 8)); + const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); + const address = t.shared.add(c.indexBase()).add(row.mul(8)); + const mapping = b.loadWords(2, address, .shared, .{}); + const page = mapping[0]; + const within = mapping[1]; + const valid = page.ge(0); + const safe_page = b.select(valid, page, b.constant(.i32, 0)); + const offset = safe_page.to(.i64).mul(c.page_size).add(within.to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + copy.close(); + } + } + + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { + const b = t.b; + var v = value; + inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); + const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); + var leader = b.openIf(t.lane.rem(4).eq(0)); + { + b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); + leader.close(); + } + // Callers synchronize once both row halves have been published. + } + + fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { + const b = t.b; + const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); + var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); + for (1..w) |i| { + const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); + result = if (maximum) result.maximum(v) else result.add(v); + } + return result; + } + + fn store(t: TileContext, row: V, dim: V, value: V) void { + const c = t.c; + var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); + if (c.splits > 1) { + offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + offset = offset.mul(c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); + } + } +}; + +fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { + std.debug.assert(c.sharedBytes() <= 99 * 1024); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .indices = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + const token = b.blockIdx().x; + const first_row = b.blockIdx().y.rem(c.headTiles()).mul(c.rows()); + const head = b.blockIdx().y.div(c.headTiles()); + const starts = b.ptrToInt(a.starts, .i64); + const active_tokens = loadI32(b, starts, b.constant(.i32, c.batch)); + // Unused capacity is observable in the public tensor and must be zero. + var inactive = b.openIf(token.ge(active_tokens)); + { + const out = b.ptrToInt(a.out, .i64); + var dims = b.openFor(b.threadIdx().x, c.rows() * (c.head_dim + (if (c.splits > 1) @as(i64, 2) else 0)), c.warps * 32, .{}); + { + const width = c.head_dim + (if (c.splits > 1) @as(i64, 2) else 0); + const row = first_row.add(dims.iv.div(width)); + const dim = dims.iv.rem(width); + var valid = b.openIf(row.lt(c.group)); + { + var offset = token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(c.group).add(row.to(.i64)); + if (c.splits > 1) { + offset = b.blockIdx().z.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(width).add(dim.to(.i64)); + const value = b.select(dim.eq(c.head_dim), b.constant(.f32, -std.math.inf(f32)), b.constant(.f32, 0)); + b.storeWords(1, out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + b.storeHalf(out.add(offset.mul(width).add(dim.to(.i64)).mul(2)), .global, b.constant(.i32, 0)); + } + valid.close(); + } + dims.close(); + } + inactive.close(); + } + var active = b.openIf(token.lt(active_tokens)); + { + const tid = b.threadIdx().x; + // Upper-bound lookup also handles inactive sequences with equal starts. + var sequence = b.openWhile(.{ b.constant(.i32, 0), b.constant(.i32, c.batch) }, .{ .i32, .i32 }); + { + sequence.yieldBefore(sequence.before_carried[0].lt(sequence.before_carried[1]), sequence.before_carried); + } + { + const left = sequence.after_carried[0]; + const right = sequence.after_carried[1]; + const mid = left.add(right).div(2); + const before = loadI32(b, starts, mid).le(token); + sequence.yieldAfter(.{ b.select(before, mid.add(1), left), b.select(before, right, mid) }); + } + const seq = sequence.results[0].sub(1); + const t: TileContext = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .indices = b.ptrToInt(a.indices, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = head, + .first_row = first_row, + .token = token, + .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), + .split = b.blockIdx().z, + }; + var copy_q = b.openFor(tid, c.rows() * (d / 8), c.warps * 32, .{}); + { + const row = copy_q.iv.div(d / 8); + const col = copy_q.iv.rem(d / 8).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(c.group); + const safe_row = b.select(valid, packed_row, b.constant(.i32, 0)); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(safe_row.to(.i64)).mul(d).add(col.to(.i64)); + b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); + copy_q.close(); + } + // Split the selected-slot list, not the logical KV timeline. Repeated + // indices remain repeated softmax columns, including across splits. + const tiles = @divTrunc(c.topk + n - 1, n); + const lo = t.split.mul(tiles).div(c.splits); + const hi = t.split.add(1).mul(tiles).div(c.splits); + const row_base = t.warp.div(w).mul(16); + const key_part = t.warp.rem(w).mul(n / w); + const dim_part = t.warp.rem(w).mul(d / w); + const row0 = t.lane.div(4); + const col0 = t.lane.rem(4).mul(2); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const nf = n / w / 8; + const of = d / w / 8; + const count = 4 + of; + var initial: [count]V = @splat(zero); + initial[0] = neg_inf; + initial[1] = neg_inf; + for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); + // Q can load while the first selected indices and pages are resolved. + var loop = b.openFor(lo, hi, 1, initial); + { + const key_base = loop.iv.mul(n); + t.gather(key_base); + t.loadKV(t.k); + b.cpAsyncWaitAll(); + b.syncThreads(); + var scores: [nf][4]V = @splat(@splat(zero)); + // Keep only MMA operands live in registers; Q remains in shared + // memory so D512 does not consume the entire per-thread register file. + var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); + { + const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); + for (0..nf) |i| { + const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); + scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); + } + qk.yield(flatten(nf, scores)); + } + for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; + var local_max: [2]V = @splat(neg_inf); + for (0..nf) |i| { + for (0..4) |j| { + const row = first_row.add(row_base).add(row0).add((j / 2) * 8); + const slot = key_part.add(i * 8).add(col0).add(j % 2); + const valid = row.lt(c.group).bitAnd(t.validSlot(slot)); + scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); + local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); + b.syncThreads(); + // All K reads have completed. Reuse its allocation for V, and + // overlap these global loads with softmax and probability stores. + t.loadKV(t.v); + var m: [2]V = undefined; + var alpha: [2]V = undefined; + var local_sum: [2]V = @splat(zero); + for (0..2) |h| { + m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); + // Empty/fully masked row tiles have m=-inf and denominator 0. + alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); + } + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + for (0..2) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + b.cpAsyncWaitAll(); + b.syncThreads(); + var next: [count]V = undefined; + for (0..2) |h| { + next[h] = m[h]; + next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); + } + var acc: [of][4]V = undefined; + for (0..of) |i| for (0..4) |j| { + acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); + }; + var packed_acc: [of]V = undefined; + for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); + var pv = b.openFor(0, n, 16, packed_acc); + { + const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + for (0..of) |i| { + const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); + var prev: [4]V = undefined; + for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); + acc[i] = mma(b, dt, pa, vb, prev); + packed_acc[i] = packAcc(b, acc[i]); + } + pv.yield(packed_acc); + } + @memcpy(next[4..], &pv.results); + b.syncThreads(); + loop.yield(next); + } + if (c.splits > tiles) b.cpAsyncWaitAll(); + for (0..2) |h| { + const row = first_row.add(row_base).add(row0).add(h * 8); + var valid = b.openIf(row.lt(c.group)); + { + const m = loop.results[h]; + const l = loop.results[h + 2]; + var norm = b.select(l.gt(0), b.rcpApprox(l), zero); + if (c.splits == 1 and c.sink) { + const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row)).bitCast(.f32).mul(std.math.log2e); + const maximum = m.maximum(sink).maximum(-std.math.floatMax(f32)); + const weight = b.exp2Approx(m.sub(maximum)); + const denominator = l.mul(weight).add(b.exp2Approx(sink.sub(maximum))); + norm = weight.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)); + } + for (0..of) |i| for (0..2) |j| { + const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); + t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); + }; + if (c.splits > 1) { + var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); + { + t.store(row, b.constant(.i32, d), m); + t.store(row, b.constant(.i32, d + 1), l); + leader.close(); + } + } + valid.close(); + } + } + active.close(); + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_indexed_grouped_merge", + .inputs = &.{ "partials", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const tile = @min(@as(i64, 64), c.head_dim); + const dim_base = b.blockIdx().z.mul(tile); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var inactive = b.openIf(token.ge(a.starts.get(.{c.batch}))); + { + var dims = b.openFor(b.threadIdx().x, tile, 64, .{}); + { + a.out.set(.{ token, head, dim_base.add(dims.iv) }, b.constant(c.dtype, 0)); + dims.close(); + } + inactive.close(); + } + var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + var maximum = b.openFor(b.threadIdx().x.rem(32), c.splits, 32, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + var maximum_value = maximum.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |step| maximum_value = maximum_value.maximum(b.shuffleXor(maximum_value, step)); + const m = maximum_value.maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, tile, 64, .{}); + { + // Expose independent loads and a balanced sum within each chunk. + // A bounded chunk avoids register spills for long selections. + const chunk = @min(@as(i64, 16), c.splits); + var sum = b.openFor(0, c.splits, chunk, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); + { + var values: [16]V = @splat(zero); + var denominators: [16]V = @splat(zero); + for (0..@as(usize, @intCast(chunk))) |i| { + const part = sum.iv.add(i); + const valid_part = part.lt(c.splits); + const safe_part = part.minimum(c.splits - 1); + const weight = b.select(valid_part, b.exp2Approx(a.partials.get(.{ safe_part, token, head, c.head_dim }).sub(m)), zero); + values[i] = weight.mul(a.partials.get(.{ safe_part, token, head, dim_base.add(dims.iv) })); + denominators[i] = weight.mul(a.partials.get(.{ safe_part, token, head, c.head_dim + 1 })); + } + var stride: usize = 1; + while (stride < 16) : (stride *= 2) { + var i: usize = 0; + while (i < 16) : (i += 2 * stride) { + values[i] = values[i].add(values[i + stride]); + denominators[i] = denominators[i].add(denominators[i + stride]); + } + } + sum.yield(.{ sum.carried[0].add(values[0]), sum.carried[1].add(denominators[0]) }); + } + a.out.set(.{ token, head, dim_base.add(dims.iv) }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); + dims.close(); + } + valid.close(); + } +} + +test "indexed grouped attention emits all supported head dimensions and dtypes" { + for ([_]i64{ 32, 64, 128, 256, 512 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| { + const cfg: Config = .{ + .head_dim = head_dim, + .tokens = 7, + .batch = 3, + .kv_heads = 2, + .group = 19, + .pages_per_seq = 64, + .page_size = 16, + .topk = 37, + .dtype = dtype, + .kv_tile = if (head_dim == 512) 32 else 64, + .scale = 1 / @sqrt(@as(f32, @floatFromInt(head_dim))), + .sink = false, + }; + try std.testing.expectEqual(@as(i64, 2), cfg.headTiles()); + try std.testing.expect(cfg.sharedBytes() <= 99 * 1024); + const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, if (dtype == .bf16) "mma.sync.aligned.m16n8k16.row.col.f32.bf16" else "mma.sync.aligned.m16n8k16.row.col.f32.f16") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cp.async") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); + }; +} + +test "indexed grouped attention emits split softmax and sink merge" { + for ([_]cute.DType{ .bf16, .f16 }) |dtype| { + const cfg: Config = .{ + .head_dim = 512, + .tokens = 7, + .batch = 3, + .kv_heads = 2, + .group = 19, + .pages_per_seq = 64, + .page_size = 16, + .topk = 257, + .dtype = dtype, + .kv_tile = 32, + .scale = 0.125, + .sink = true, + .splits = 3, + }; + const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + const merge_ir = try Merge.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); + defer std.testing.allocator.free(merge_ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x2.trans") != null); + try std.testing.expect(std.mem.indexOf(u8, merge_ir, "scf.for") != null); + } +} + +test "indexed grouped attention emits single warp D32 D64 D128" { + for ([_]i64{ 32, 64, 128 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| for ([_]i64{ 16, 32, 64 }) |tile| { + const cfg: Config = .{ + .head_dim = head_dim, + .tokens = 7, + .batch = 3, + .kv_heads = 2, + .group = 19, + .pages_per_seq = 64, + .page_size = 16, + .topk = 65, + .dtype = dtype, + .kv_tile = tile, + .warps = 1, + .scale = 0.125, + .sink = true, + .splits = 3, + }; + const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "mma.sync.aligned.m16n8k16") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x2.trans") != null); + }; +} diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 4a97a38e..9f640c8d 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -10,7 +10,7 @@ const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); -const triton = @import("triton/triton.zig"); +pub const triton = @import("triton/triton.zig"); const PagedAttention = @This(); From 25629637e642de55dc38f7e01ba3226a07fbb659 Mon Sep 17 00:00:00 2001 From: dmehala Date: Sun, 15 Nov 2026 09:32:11 +0000 Subject: [PATCH 2/3] wip --- .../paged_attention/cute/benchmark.zig | 80 ++- zml/kernels/paged_attention/cute/cases.zig | 7 + .../paged_attention/cute/correctness_test.zig | 14 + zml/kernels/paged_attention/cute/cute.zig | 39 +- .../paged_attention/cute/sparse_attention.zig | 611 ++++++++++++------ 5 files changed, 553 insertions(+), 198 deletions(-) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index dafd2c81..2ccb2955 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -7,6 +7,7 @@ const Inputs = fixtures.Inputs; const Args = struct { profile_dir: ?[]const u8 = null, + paired: bool = false, sparse: bool = false, stress: bool = false, all_backends: bool = false, @@ -41,6 +42,10 @@ pub fn main(init: std.process.Init) !void { } else if (!std.mem.eql(u8, filter, c.name)) continue; } selected += 1; + if (args.paired) { + try runPaired(init.gpa, init.io, platform, mesh, c, index, args); + continue; + } const backends = std.enums.values(fixtures.Backend); for (0..backends.len) |rank| { const backend = backends[(rank + index) % backends.len]; @@ -73,6 +78,8 @@ fn parseArgs(init: std.process.Init) !Args { while (it.next()) |arg| { if (std.mem.eql(u8, arg, "--sparse")) { result.sparse = true; + } else if (std.mem.eql(u8, arg, "--paired")) { + result.paired = true; } else if (std.mem.eql(u8, arg, "--stress")) { result.sparse = true; result.stress = true; @@ -93,10 +100,11 @@ fn parseArgs(init: std.process.Init) !Args { result.samples = try std.fmt.parseInt(usize, arg["--samples=".len..], 10); if (result.samples == 0) return error.InvalidSamples; } else if (std.mem.eql(u8, arg, "--help")) { - std.debug.print("paged attention benchmark [--sparse|--stress] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); + std.debug.print("paged attention benchmark [--sparse|--stress] [--paired] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); std.process.exit(0); } else return error.UnknownArgument; } + if (result.paired and (result.stress or result.backend != null or result.profile_dir != null)) return error.InvalidPairedOptions; return result; } @@ -201,3 +209,73 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtur fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } + +// Keep all executables and inputs alive and rotate order for each sample round. +fn runPaired(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case, index: usize, args: Args) !void { + var global_case = c; + global_case.kv_heads *= platform.devices.len; + var inputs = Inputs.initDType(global_case, args.dtype); + inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); + inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); + inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); + inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); + if (args.dtype == .f16) fixture.quantizeF16(); + var buffers = try fixtures.upload(a, io, platform, inputs, fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + const backends = std.enums.values(fixtures.Backend); + var executables: [3]zml.Exe = undefined; + var runners: [3]zml.Exe.Runner = undefined; + var compiled: usize = 0; + var initialized: usize = 0; + defer for (executables[0..compiled]) |e| e.deinit(); + defer for (runners[0..initialized]) |*r| r.deinit(a); + var errors: [3]f32 = undefined; + var compile_us: [3]f64 = undefined; + var samples: [3][]f64 = undefined; + for (&samples) |*v| v.* = try arena.allocator().alloc(f64, args.samples); + for (backends, 0..) |backend, i| { + const start: std.Io.Timestamp = .now(io, .awake); + executables[i] = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend }, .{ .program_name = "paired_attention", .meshes = &.{mesh} }); + compiled += 1; + compile_us[i] = elapsedUs(start, io); + runners[i] = try executables[i].runner(a); + initialized += 1; + var output: zml.Buffer = undefined; + runners[i].run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + errors[i] = if (args.dtype == .f16) try reference.check(a, global_case, fixture, host.items(f16)) else try reference.check(a, global_case, fixture, host.items(zml.floats.BFloat16)); + } + for (0..args.warmup + args.samples) |round| { + for (0..backends.len) |rank| { + const i = (rank + round) % backends.len; + var output: zml.Buffer = undefined; + const start: std.Io.Timestamp = .now(io, .awake); + runners[i].run(io, .{buffers}, .{&output}, .{ .wait = true }); + const us = elapsedUs(start, io); + if (round >= args.warmup) samples[i][round - args.warmup] = us; + output.deinit(); + } + } + const deltas = try arena.allocator().alloc(f64, args.samples); + const ratios = try arena.allocator().alloc(f64, args.samples); + const cute_index = @backingInt(fixtures.Backend.cute); + const triton_index = @backingInt(fixtures.Backend.triton); + for (deltas, ratios, 0..) |*delta, *ratio, i| { + delta.* = samples[cute_index][i] - samples[triton_index][i]; + ratio.* = samples[triton_index][i] / samples[cute_index][i]; + } + std.mem.sort(f64, deltas, {}, std.sort.asc(f64)); + std.mem.sort(f64, ratios, {}, std.sort.asc(f64)); + std.debug.print("PAIR index={d} compute_dtype={s} n={d} cute_minus_triton_median_us={d:.3} triton_over_cute_median={d:.6} delta_p10_us={d:.3} delta_p90_us={d:.3}\n", .{ index, @tagName(args.dtype), deltas.len, deltas[deltas.len / 2], ratios[ratios.len / 2], deltas[deltas.len / 10], deltas[(deltas.len - 1) * 9 / 10] }); + for (backends, 0..) |backend, i| { + const values = samples[i]; + std.mem.sort(f64, values, {}, std.sort.asc(f64)); + const median = if (values.len % 2 == 0) (values[values.len / 2 - 1] + values[values.len / 2]) / 2 else values[values.len / 2]; + std.debug.print("RESULT index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_outputs paired=true max_abs={d:.6} compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(args.dtype), errors[i], compile_us[i] / 1000, values.len, median, values[values.len / 10], values[(values.len - 1) * 9 / 10] }); + } +} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index 0c2f4c04..35c7c38f 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -108,6 +108,13 @@ pub const benchmarks = blk: { // Selection-slot tails, GQA head tails, and logical-to-physical page gathers // vary independently. Every nonempty fixture includes duplicate/invalid slots. pub const sparseCorrectness = [_]Case{ + .{ .name = "sparse_decode_inactive", .head_dim = 64, .group = 5, .batch = 5, .inactive = 2, .token_capacity = 9, .mask = .indices, .topk = 65, .shuffle_pages = true }, + .{ .name = "sparse_tiny_decode_inactive", .head_dim = 128, .group = 1, .batch = 5, .inactive = 2, .token_capacity = 9, .mask = .indices, .topk = 8, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_tiny_d32_sink", .head_dim = 32, .group = 1, .mask = .indices, .topk = 1, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_tiny_d64_disabled_sink", .head_dim = 64, .group = 1, .mask = .indices, .topk = 7, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, + .{ .name = "sparse_tiny_d128_empty", .head_dim = 128, .group = 1, .mask = .indices, .topk = 8, .empty_indices = true }, + .{ .name = "sparse_tiny_d256_capacity", .head_dim = 256, .group = 1, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .mask = .indices, .topk = 5, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_tiny_d512_scale", .head_dim = 512, .group = 1, .mask = .indices, .topk = 8, .scale = -0.125, .shuffle_pages = true }, .{ .name = "sparse_tiny_mha", .head_dim = 128, .group = 1, .mask = .indices, .topk = 8, .shuffle_pages = true }, .{ .name = "sparse_d32_mha", .head_dim = 32, .group = 1, .mask = .indices, .topk = 32 }, .{ .name = "sparse_d64_mqa", .head_dim = 64, .kv_heads = 1, .group = 8, .mask = .indices, .topk = 65 }, diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index deb4dbf8..b1bf3b8b 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -55,6 +55,20 @@ test "indexed paged attention FP16 all backends" { try runSparseCorrectness(.f16); } +test "indexed long selections handle sinks tails and inactive capacity" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + const suite = [_]cases.Case{ + .{ .name = "sparse_wide_sink_tail", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 16, .kv_len = 2049, .mask = .indices, .topk = 1025, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_wide_empty_capacity", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 5, .inactive = 1, .token_capacity = 6, .mask = .indices, .topk = 513, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, + .{ .name = "sparse_wide_prefill", .head_dim = 512, .group = 3, .prefill = 1, .chunk = 3, .kv_len = 2049, .mask = .indices, .topk = 513, .sink = true, .shuffle_pages = true }, + }; + for ([_]zml.DataType{ .bf16, .f16 }) |dtype| for (suite) |c| for (std.enums.values(fixtures.Backend)) |backend| { + std.debug.print("CASE name={s} backend={s} compute_dtype={s}\n", .{ c.name, @tagName(backend), @tagName(dtype) }); + try runCorrectness(std.testing.allocator, std.testing.io, platform, c, backend, dtype); + }; +} + fn runSparseCorrectness(dtype: zml.DataType) !void { const platform = zml.testing.env(); if (platform.target != .cuda) return error.CudaRequired; diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 6d30693a..8cd2b659 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -140,9 +140,10 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); const indices = sparseIndices(q_, opts.mask.indices); - // A short, single-head selection has no KV reuse to amortize MMA tiling. - if (q_.dim(.hg) == 1 and indices.dim(.topk) <= 8) - return pagedAttentionGeneric(parameters, q, k, v, opts); + const all_decode = !parameters.options_.is_prefill; + const short_selection = all_decode and q_.dim(.hd) <= 128 and indices.dim(.topk) <= 128; + const fine_tile = all_decode and q_.dim(.b) == 1 and ((q_.dim(.hd) == 512 and indices.dim(.topk) <= 512) or (q_.dim(.hd) >= 128 and q_.dim(.hd) < 512 and q_.dim(.hg) <= 16)); + const split_features = all_decode and (q_.dim(.hd) == 512 or (fine_tile and indices.dim(.topk) > 128)); var cfg: sparse.Config = .{ .head_dim = q_.dim(.hd), .tokens = q_.dim(.b), @@ -153,20 +154,36 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: .page_size = k_.dim(.k_chunk), .topk = indices.dim(.topk), .dtype = zml.kernel.cute.from(q_.dtype()), - .kv_tile = if (!parameters.options_.is_prefill and q_.dim(.hd) <= 256) 16 else if (q_.dim(.hd) == 512) 32 else 64, - .warps = if (!parameters.options_.is_prefill and q_.dim(.hd) <= 256) @max(1, @divTrunc(q_.dim(.hd), 128)) else 4, + .kv_tile = if (short_selection) (if (indices.dim(.topk) <= 32) @as(i64, 32) else 128) else if (fine_tile) 16 else if (all_decode or q_.dim(.hd) == 512) 32 else 64, + .warps = if (split_features) 8 else 4, .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), .sink = opts.sink != null, + .decode = all_decode and parameters.options_.max_seqlen_q == 1, }; + cfg.merge_parts = if (cfg.head_dim >= 128 and cfg.head_dim <= 256 and cfg.tokens * cfg.kv_heads * cfg.group <= 16) 16 else 8; const work = @max(1, cfg.tokens * cfg.kv_heads * cfg.headTiles()); - // Narrow decode CTAs keep about 64 FP32 output registers per thread. - // One selected tile per partition exposes parallelism at small batches. + // Short selections avoid a merge; longer decode selections expose one + // selected tile per partition until the device has enough parallel work. const target = @as(i64, @intCast(multiprocessorCount(zml.Compiler.current().platform))) * 16; const tiles = @divTrunc(cfg.topk + cfg.kv_tile - 1, cfg.kv_tile); var desired: i64 = 1; while (desired * work < target) desired *= 2; - cfg.splits = if (!parameters.options_.is_prefill) @max(1, @min(128, @min(tiles, desired))) else if (cfg.head_dim <= 128) 1 else @max(1, @min(32, @min(@divTrunc(@divTrunc(target, 8) + work - 1, work), @divTrunc(cfg.topk, 128)))); + cfg.splits = if (short_selection) 1 else if (all_decode) @max(1, @min(128, @min(tiles, desired))) else if (cfg.head_dim <= 128) 1 else @max(1, @min(32, @min(@divTrunc(@divTrunc(target, 8) + work - 1, work), @divTrunc(cfg.topk, 128)))); const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.scalar(0, .f32); + if (cfg.group == 1 and cfg.topk <= 8) return sparse.Tiny.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads), 1 }, + .block = .{ if (cfg.topk > 4) @as(i32, 256) else 128, 1, 1 }, + }).out.transpose(q.shape()); const out = sparse.Kernel.call(.{ .q = q_, .k = k_, @@ -176,15 +193,15 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: .starts = parameters.query_start_len, .sink = sink, .indices = indices, - }, .{ .out = if (cfg.splits > 1) zml.Shape.init(.{ cfg.splits, cfg.tokens, cfg.kv_heads * cfg.group, cfg.head_dim + 2 }, .f32) else q_.shape() }, .{ + }, .{ .out = if (cfg.splits > 1) zml.Shape.init(.{ cfg.tokens, cfg.kv_heads * cfg.group, cfg.splits, cfg.partialWidth() }, .f32) else q_.shape() }, .{ .cfg = cfg, .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.headTiles()), @intCast(cfg.splits) }, .block = .{ @intCast(cfg.warps * 32), 1, 1 }, }).out; const result = if (cfg.splits > 1) sparse.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divTrunc(cfg.head_dim + 63, 64)) }, - .block = .{ 64, 1, 1 }, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divTrunc(cfg.head_dim, @divExact(128, cfg.merge_parts))) }, + .block = .{ 128, 1, 1 }, }).out else out; return result.transpose(q.shape()); } diff --git a/zml/kernels/paged_attention/cute/sparse_attention.zig b/zml/kernels/paged_attention/cute/sparse_attention.zig index 3d15634f..f9021041 100644 --- a/zml/kernels/paged_attention/cute/sparse_attention.zig +++ b/zml/kernels/paged_attention/cute/sparse_attention.zig @@ -24,6 +24,8 @@ pub const Config = struct { warps: i64 = 4, scale: f32, sink: bool, + decode: bool = false, + merge_parts: i64 = 8, splits: i64 = 1, pub fn rows(c: Config) i64 { @@ -36,11 +38,18 @@ pub const Config = struct { fn kvBase(c: Config) i64 { return c.rows() * c.head_dim * 2; } + pub fn partialWidth(c: Config) i64 { + // Align each split to a 32-byte sector, including max/denominator. + return c.head_dim + 8; + } fn pBase(c: Config) i64 { return c.kvBase() + c.kv_tile * c.head_dim * 2; } fn redBase(c: Config) i64 { - return c.pBase() + c.rows() * c.kv_tile * 2; + const key_warps = @min(c.warps, @min(@divTrunc(c.kv_tile, 8), 4)); + const feature_parts = @divExact(c.warps, key_warps); + // Wide-head QK partials share this allocation with the later FP16 P. + return c.pBase() + c.rows() * c.kv_tile * (if (feature_parts > 1) feature_parts * 4 else 2); } fn indexBase(c: Config) i64 { return c.redBase() + 2 * 64 * 4; @@ -57,6 +66,145 @@ pub const Kernel = cute.Kernel(Config, .{ .run = run, }); +// Small MHA selections use SIMT: four warps evaluate independent selected +// slots, then reduce their weighted values within one CTA and one launch. +pub const Tiny = cute.Kernel(Config, .{ + .name = "paged_attention_cute_indexed_tiny", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, + .outputs = &.{"out"}, + .run = runTiny, +}); + +// In a decode-only batch each sequence owns zero or one query. Equal total +// active tokens and sequence count proves token == sequence; spare capacity +// and inactive sequences retain the general upper-bound lookup. +fn findSequence(b: *B, c: Config, starts: V, token: V, active_tokens: V) V { + var fast = b.openIfElse(if (c.decode) active_tokens.eq(c.batch) else b.constant(.i1, 0), .{.i32}); + fast.yieldThen(.{token}); + var sequence = b.openWhile(.{ b.constant(.i32, 0), b.constant(.i32, c.batch) }, .{ .i32, .i32 }); + sequence.yieldBefore(sequence.before_carried[0].lt(sequence.before_carried[1]), sequence.before_carried); + const left = sequence.after_carried[0]; + const right = sequence.after_carried[1]; + const mid = left.add(right).div(2); + const before = loadI32(b, starts, mid).le(token); + sequence.yieldAfter(.{ b.select(before, mid.add(1), left), b.select(before, right, mid) }); + fast.yieldElse(.{sequence.results[0].sub(1)}); + return fast.results[0]; +} + +fn loadHalf(b: *B, address: V, comptime dt: cute.DType) V { + return b.emit(llvm.load(b.ctx, b.addressPtr(address, .global).inner, dt.toMlir(b.ctx), .{ .alignment = 2 }, b.loc())).to(.f32); +} + +fn runTiny(b: *B, c: Config) cute.FinishError!void { + std.debug.assert(c.group == 1 and c.topk <= 8); + switch (c.head_dim) { + inline 32, 64, 128, 256, 512 => |d| switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTinyDimension(b, c, d, dt), + else => unreachable, + }, + else => unreachable, + } +} + +fn runTinyDimension(b: *B, c: Config, comptime d: usize, comptime dt: cute.DType) cute.FinishError!void { + const chunks = d / 32; + const warps: usize = if (c.topk > 4) 8 else 4; + const slots: usize = 8 / warps; + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .indices = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = tid.div(32); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const starts = b.ptrToInt(a.starts, .i64); + const active_tokens = loadI32(b, starts, b.constant(.i32, c.batch)); + const out = b.ptrToInt(a.out, .i64).add(token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2)); + var inactive = b.openIf(token.ge(active_tokens)); + { + var dims = b.openFor(tid, d, warps * 32, .{}); + b.storeHalf(out.add(dims.iv.to(.i64).mul(2)), .global, b.constant(.i32, 0)); + dims.close(); + inactive.close(); + } + var active = b.openIf(token.lt(active_tokens)); + { + const seq = findSequence(b, c, starts, token, active_tokens); + const len = loadI32(b, b.ptrToInt(a.lengths, .i64), seq); + const q = b.ptrToInt(a.q, .i64).add(token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2)); + const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(32 + warps * d * 4), 16), .i32); + var query: [chunks]V = undefined; + for (&query, 0..) |*value, i| value.* = loadHalf(b, q.add(lane.add(i * 32).to(.i64).mul(2)), dt); + var rows: [2]V = undefined; + var valid: [2]V = undefined; + for (0..slots) |i| { + const slot = warp.add(i * warps); + var selected = b.openIfElse(slot.lt(c.topk), .{.i32}); + selected.yieldThen(.{loadI32(b, b.ptrToInt(a.indices, .i64), token.to(.i64).mul(c.topk).add(slot.to(.i64)))}); + selected.yieldElse(.{b.constant(.i32, -1)}); + const key = selected.results[0]; + valid[i] = key.ge(0).bitAnd(key.lt(len)); + var entry = b.openIfElse(valid[i], .{ .f32, .i64 }); + { + const page = loadI32(b, b.ptrToInt(a.table, .i64), seq.mul(c.pages_per_seq).add(key.div(c.page_size))); + const row = page.to(.i64).mul(c.page_size).add(key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2); + var score = zero; + for (query, 0..) |value, j| score = score.add(value.mul(loadHalf(b, b.ptrToInt(a.k, .i64).add(row).add(lane.add(j * 32).to(.i64).mul(2)), dt))); + inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); + entry.yieldThen(.{ score.mul(c.scale * std.math.log2e), row }); + } + entry.yieldElse(.{ neg_inf, b.constant(.i64, 0) }); + rows[i] = entry.results[1]; + var leader = b.openIf(lane.eq(0)); + b.storeWords(1, shared.add(slot.mul(4)), .shared, .{entry.results[0].bitCast(.i32)}); + leader.close(); + } + b.syncThreads(); + const sink = if (c.sink) b.loadWords(1, b.ptrToInt(a.sink, .i64).add(head.to(.i64).mul(4)), .global, .{})[0].bitCast(.f32).mul(std.math.log2e) else neg_inf; + var maximum = sink; + var scores: [8]V = undefined; + for (&scores, 0..) |*score, i| { + score.* = b.loadWords(1, shared.add(i * 4), .shared, .{})[0].bitCast(.f32); + maximum = maximum.maximum(score.*); + } + maximum = maximum.maximum(b.constant(.f32, -std.math.floatMax(f32))); + var denominator = if (c.sink) b.exp2Approx(sink.sub(maximum)) else zero; + for (scores) |score| denominator = denominator.add(b.exp2Approx(score.sub(maximum))); + const norm = b.select(denominator.gt(0), b.rcpApprox(denominator), zero); + var output: [chunks]V = @splat(zero); + for (0..slots) |i| { + const score = b.loadWords(1, shared.add(warp.add(i * warps).mul(4)), .shared, .{})[0].bitCast(.f32); + const weight = b.exp2Approx(score.sub(maximum)).mul(norm); + var entry = b.openIfElse(valid[i], @as([chunks]V, @splat(zero))); + var values: [chunks]V = undefined; + for (&values, 0..) |*value, j| value.* = output[j].add(weight.mul(loadHalf(b, b.ptrToInt(a.v, .i64).add(rows[i]).add(lane.add(j * 32).to(.i64).mul(2)), dt))); + entry.yieldThen(values); + entry.yieldElse(output); + output = entry.results; + } + for (output, 0..) |value, i| b.storeWords(1, shared.add(32).add(warp.mul(d).add(lane.add(i * 32)).mul(4)), .shared, .{value.bitCast(.i32)}); + b.syncThreads(); + var dims = b.openFor(tid, d, warps * 32, .{}); + var value = zero; + for (0..warps) |i| value = value.add(b.loadWords(1, shared.add(32 + i * d * 4).add(dims.iv.mul(4)), .shared, .{})[0].bitCast(.f32)); + b.storeHalf(out.add(dims.iv.to(.i64).mul(2)), .global, value.to(dt).bitCast(.i16).to(.i32)); + dims.close(); + active.close(); + } +} + fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { var result: [n * 4]V = undefined; for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); @@ -100,12 +248,16 @@ fn run(b: *B, c: Config) cute.FinishError!void { switch (c.head_dim) { inline 32, 64, 128, 256 => |d| switch (c.warps) { inline 1, 2, 4 => |w| try runDimension(b, c, d, w), + 8 => if (d >= 64) try runDimension(b, c, d, 8) else unreachable, else => unreachable, }, 512 => { const d = 512; - std.debug.assert(c.warps == 4); - try runDimension(b, c, d, 4); + switch (c.warps) { + 4 => try runDimension(b, c, d, 4), + 8 => try runDimension(b, c, d, 8), + else => unreachable, + } }, else => unreachable, } @@ -113,11 +265,11 @@ fn run(b: *B, c: Config) cute.FinishError!void { fn runDimension(b: *B, c: Config, comptime d: usize, comptime w: usize) cute.FinishError!void { switch (c.kv_tile) { - 16 => if (w <= 2) switch (c.dtype) { + 16 => switch (c.dtype) { inline .bf16, .f16 => |dt| try runTile(b, c, d, w, 16, dt), else => unreachable, - } else unreachable, - inline 32, 64 => |n| switch (c.dtype) { + }, + inline 32, 64, 128 => |n| switch (c.dtype) { inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), else => unreachable, }, @@ -146,6 +298,12 @@ const TileContext = struct { klen: V, split: V, + fn sync(t: TileContext) void { + if (t.c.warps == 1) { + t.b.syncWarp(); + } else t.b.syncThreads(); + } + // Resolve each selected slot once, then share its physical page across // every head, feature vector, and both K/V loads. Invalid logical indices // never dereference the page table; cp.async zero-fills their K/V rows. @@ -172,7 +330,7 @@ const TileContext = struct { } selected.close(); } - b.syncThreads(); + t.sync(); } fn validSlot(t: TileContext, row: V) V { @@ -180,24 +338,61 @@ const TileContext = struct { } fn loadKV(t: TileContext, source: V) void { + const vectors = @divExact(t.c.kv_tile * t.c.head_dim, 8); + const threads = t.c.warps * 32; + for (0..@intCast(@divTrunc(vectors + threads - 1, threads))) |i| { + const iv = t.tid.add(i * @as(usize, @intCast(threads))); + if (@mod(vectors, threads) == 0) { + t.copyKV(source, iv); + } else { + var valid = t.b.openIf(iv.lt(vectors)); + t.copyKV(source, iv); + valid.close(); + } + } + } + + fn copyKV(t: TileContext, source: V, iv: V) void { const b = t.b; const c = t.c; - var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), c.warps * 32, .{}); - { - const row = copy.iv.div(@divExact(c.head_dim, 8)); - const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); - const address = t.shared.add(c.indexBase()).add(row.mul(8)); - const mapping = b.loadWords(2, address, .shared, .{}); - const page = mapping[0]; - const within = mapping[1]; - const valid = page.ge(0); - const safe_page = b.select(valid, page, b.constant(.i32, 0)); - const offset = safe_page.to(.i64).mul(c.page_size).add(within.to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); - b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); - copy.close(); + const row = iv.div(@divExact(c.head_dim, 8)); + const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); + const address = t.shared.add(c.indexBase()).add(row.mul(8)); + const mapping = b.loadWords(2, address, .shared, .{}); + const page = mapping[0]; + const within = mapping[1]; + const valid = page.ge(0); + const safe_page = b.select(valid, page, b.constant(.i32, 0)); + const offset = safe_page.to(.i64).mul(c.page_size).add(within.to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + } + + fn loadQuery(t: TileContext) void { + const vectors = @divExact(t.c.rows() * t.c.head_dim, 8); + const threads = t.c.warps * 32; + for (0..@intCast(@divTrunc(vectors + threads - 1, threads))) |i| { + const iv = t.tid.add(i * @as(usize, @intCast(threads))); + if (@mod(vectors, threads) == 0) { + t.copyQuery(iv); + } else { + var valid = t.b.openIf(iv.lt(vectors)); + t.copyQuery(iv); + valid.close(); + } } } + fn copyQuery(t: TileContext, iv: V) void { + const c = t.c; + const row = iv.div(@divExact(c.head_dim, 8)); + const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); + const packed_row = t.first_row.add(row); + const valid = packed_row.lt(c.group); + const safe_row = t.b.select(valid, packed_row, t.b.constant(.i32, 0)); + const offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(safe_row.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + t.b.cpAsync16(t.shared.add(swizzle(row, col, c.head_dim)), t.q.add(offset.mul(2)), valid); + } + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { const b = t.b; var v = value; @@ -213,7 +408,7 @@ const TileContext = struct { fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { const b = t.b; - const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); + const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(row.mul(4)); var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); for (1..w) |i| { const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); @@ -226,7 +421,7 @@ const TileContext = struct { const c = t.c; var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); if (c.splits > 1) { - offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); + offset = offset.mul(c.splits).add(t.split.to(.i64)).mul(c.partialWidth()).add(dim.to(.i64)); t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); } else { offset = offset.mul(c.head_dim).add(dim.to(.i64)); @@ -248,7 +443,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u .indices = .{ .ptr = cute.DType.i32 }, .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, }); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + if (c.sharedBytes() > 48 * 1024) b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", .int(b.ctx, .i32, c.sharedBytes()))}))); const token = b.blockIdx().x; const first_row = b.blockIdx().y.rem(c.headTiles()).mul(c.rows()); const head = b.blockIdx().y.div(c.headTiles()); @@ -258,16 +453,16 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u var inactive = b.openIf(token.ge(active_tokens)); { const out = b.ptrToInt(a.out, .i64); - var dims = b.openFor(b.threadIdx().x, c.rows() * (c.head_dim + (if (c.splits > 1) @as(i64, 2) else 0)), c.warps * 32, .{}); + var dims = b.openFor(b.threadIdx().x, c.rows() * (if (c.splits > 1) c.partialWidth() else c.head_dim), c.warps * 32, .{}); { - const width = c.head_dim + (if (c.splits > 1) @as(i64, 2) else 0); + const width = if (c.splits > 1) c.partialWidth() else c.head_dim; const row = first_row.add(dims.iv.div(width)); const dim = dims.iv.rem(width); var valid = b.openIf(row.lt(c.group)); { var offset = token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(c.group).add(row.to(.i64)); if (c.splits > 1) { - offset = b.blockIdx().z.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(width).add(dim.to(.i64)); + offset = offset.mul(c.splits).add(b.blockIdx().z.to(.i64)).mul(width).add(dim.to(.i64)); const value = b.select(dim.eq(c.head_dim), b.constant(.f32, -std.math.inf(f32)), b.constant(.f32, 0)); b.storeWords(1, out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); } else { @@ -282,19 +477,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u var active = b.openIf(token.lt(active_tokens)); { const tid = b.threadIdx().x; - // Upper-bound lookup also handles inactive sequences with equal starts. - var sequence = b.openWhile(.{ b.constant(.i32, 0), b.constant(.i32, c.batch) }, .{ .i32, .i32 }); - { - sequence.yieldBefore(sequence.before_carried[0].lt(sequence.before_carried[1]), sequence.before_carried); - } - { - const left = sequence.after_carried[0]; - const right = sequence.after_carried[1]; - const mid = left.add(right).div(2); - const before = loadI32(b, starts, mid).le(token); - sequence.yieldAfter(.{ b.select(before, mid.add(1), left), b.select(before, right, mid) }); - } - const seq = sequence.results[0].sub(1); + const seq = findSequence(b, c, starts, token, active_tokens); const t: TileContext = .{ .b = b, .c = c, @@ -316,128 +499,39 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), .split = b.blockIdx().z, }; - var copy_q = b.openFor(tid, c.rows() * (d / 8), c.warps * 32, .{}); - { - const row = copy_q.iv.div(d / 8); - const col = copy_q.iv.rem(d / 8).mul(8); - const packed_row = first_row.add(row); - const valid = packed_row.lt(c.group); - const safe_row = b.select(valid, packed_row, b.constant(.i32, 0)); - const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(safe_row.to(.i64)).mul(d).add(col.to(.i64)); - b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); - copy_q.close(); - } + t.loadQuery(); // Split the selected-slot list, not the logical KV timeline. Repeated // indices remain repeated softmax columns, including across splits. const tiles = @divTrunc(c.topk + n - 1, n); const lo = t.split.mul(tiles).div(c.splits); const hi = t.split.add(1).mul(tiles).div(c.splits); const row_base = t.warp.div(w).mul(16); - const key_part = t.warp.rem(w).mul(n / w); const dim_part = t.warp.rem(w).mul(d / w); const row0 = t.lane.div(4); const col0 = t.lane.rem(4).mul(2); const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const nf = n / w / 8; const of = d / w / 8; const count = 4 + of; var initial: [count]V = @splat(zero); initial[0] = neg_inf; initial[1] = neg_inf; for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); - // Q can load while the first selected indices and pages are resolved. - var loop = b.openFor(lo, hi, 1, initial); - { - const key_base = loop.iv.mul(n); - t.gather(key_base); - t.loadKV(t.k); - b.cpAsyncWaitAll(); - b.syncThreads(); - var scores: [nf][4]V = @splat(@splat(zero)); - // Keep only MMA operands live in registers; Q remains in shared - // memory so D512 does not consume the entire per-thread register file. - var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); - { - const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); - for (0..nf) |i| { - const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); - scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); - } - qk.yield(flatten(nf, scores)); - } - for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; - var local_max: [2]V = @splat(neg_inf); - for (0..nf) |i| { - for (0..4) |j| { - const row = first_row.add(row_base).add(row0).add((j / 2) * 8); - const slot = key_part.add(i * 8).add(col0).add(j % 2); - const valid = row.lt(c.group).bitAnd(t.validSlot(slot)); - scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); - local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); - } - } - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); - b.syncThreads(); - // All K reads have completed. Reuse its allocation for V, and - // overlap these global loads with softmax and probability stores. - t.loadKV(t.v); - var m: [2]V = undefined; - var alpha: [2]V = undefined; - var local_sum: [2]V = @splat(zero); - for (0..2) |h| { - m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); - // Empty/fully masked row tiles have m=-inf and denominator 0. - alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); - } - for (0..nf) |i| { - for (0..4) |j| { - scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); - local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); - } - for (0..2) |h| { - const row = row_base.add(row0).add(h * 8); - const col = key_part.add(i * 8).add(col0); - b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); - } - } - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); - b.cpAsyncWaitAll(); - b.syncThreads(); - var next: [count]V = undefined; - for (0..2) |h| { - next[h] = m[h]; - next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); - } - var acc: [of][4]V = undefined; - for (0..of) |i| for (0..4) |j| { - acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); - }; - var packed_acc: [of]V = undefined; - for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); - var pv = b.openFor(0, n, 16, packed_acc); - { - const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); - for (0..of) |i| { - const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); - var prev: [4]V = undefined; - for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); - acc[i] = mma(b, dt, pa, vb, prev); - packed_acc[i] = packAcc(b, acc[i]); - } - pv.yield(packed_acc); - } - @memcpy(next[4..], &pv.results); - b.syncThreads(); - loop.yield(next); - } + // A one-tile partition needs neither online rescaling nor a tile loop. + const result = if (c.splits == tiles) + advanceTile(t, d, w, n, dt, true, t.split.mul(n), initial) + else blk: { + var loop = b.openFor(lo, hi, 1, initial); + loop.yield(advanceTile(t, d, w, n, dt, false, loop.iv.mul(n), loop.carried)); + break :blk loop.results; + }; if (c.splits > tiles) b.cpAsyncWaitAll(); for (0..2) |h| { const row = first_row.add(row_base).add(row0).add(h * 8); var valid = b.openIf(row.lt(c.group)); { - const m = loop.results[h]; - const l = loop.results[h + 2]; + const m = result[h]; + const l = result[h + 2]; var norm = b.select(l.gt(0), b.rcpApprox(l), zero); if (c.splits == 1 and c.sink) { const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row)).bitCast(.f32).mul(std.math.log2e); @@ -447,7 +541,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u norm = weight.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)); } for (0..of) |i| for (0..2) |j| { - const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); + const value = b.vectorExtract(result[4 + i], @intCast(h * 2 + j), .f32); t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); }; if (c.splits > 1) { @@ -465,6 +559,141 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u } } +fn advanceTile(t: TileContext, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType, comptime single: bool, key_base: V, previous: [4 + d / w / 8]V) [4 + d / w / 8]V { + const b = t.b; + const c = t.c; + const first_row = t.first_row; + const row_base = t.warp.div(w).mul(16); + const key_warps = @min(w, @min(n / 8, 4)); + const feature_parts = w / key_warps; + const feature_part = t.warp.div(key_warps).mul(d / feature_parts); + const key_part = t.warp.rem(key_warps).mul(n / key_warps); + const dim_part = t.warp.rem(w).mul(d / w); + const row0 = t.lane.div(4); + const col0 = t.lane.rem(4).mul(2); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const nf = n / key_warps / 8; + const of = d / w / 8; + const count = 4 + of; + t.gather(key_base); + t.loadKV(t.k); + b.cpAsyncWaitAll(); + t.sync(); + var scores: [nf][4]V = @splat(@splat(zero)); + // Keep only MMA operands live in registers; Q remains in shared + // memory so D512 does not consume the entire per-thread register file. + for (0..d / feature_parts / 16) |ki| { + const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), feature_part.add(ki * 16).add(t.lane.div(16).mul(8)), d))); + for (0..nf) |i| { + const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), feature_part.add(ki * 16).add(t.lane.div(8).rem(2).mul(8)), d))); + scores[i] = mma(b, dt, qa, kb, scores[i]); + } + } + if (feature_parts > 1) { + // Store each MMA register as a contiguous warp vector; row-major + // FP32 score tiles alias shared-memory banks across query rows. + // The max-reduction barrier protects these reads before reuse as P. + const bank = t.warp.div(key_warps); + for (0..nf) |i| for (0..4) |j| { + const register = t.warp.rem(key_warps).mul(nf * 128).add(i * 128 + j * 32).add(t.lane); + const offset = bank.mul(16 * n).add(register).mul(4); + b.storeWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{scores[i][j].bitCast(.i32)}); + }; + t.sync(); + for (0..nf) |i| for (0..4) |j| { + const register = t.warp.rem(key_warps).mul(nf * 128).add(i * 128 + j * 32).add(t.lane); + for (1..feature_parts) |peer| { + const offset = bank.add(peer).rem(feature_parts).mul(16 * n).add(register).mul(4); + scores[i][j] = scores[i][j].add(b.loadWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{})[0].bitCast(.f32)); + } + }; + } + var local_max: [2]V = @splat(neg_inf); + for (0..nf) |i| { + for (0..4) |j| { + const row = first_row.add(row_base).add(row0).add((j / 2) * 8); + const slot = key_part.add(i * 8).add(col0).add(j % 2); + const valid = row.lt(c.group).bitAnd(t.validSlot(slot)); + scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); + local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); + } + } + if (w == 1) { + for (&local_max) |*value| inline for (.{ 2, 1 }) |step| { + value.* = value.*.maximum(b.shuffleXor(value.*, step)); + }; + } else { + var unique = b.openIf(t.warp.lt(key_warps)); + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); + unique.close(); + t.sync(); + } + // All K reads have completed. Reuse its allocation for V, and + // overlap these global loads with softmax and probability stores. + t.loadKV(t.v); + var m: [2]V = undefined; + var alpha: [2]V = undefined; + var local_sum: [2]V = @splat(zero); + for (0..2) |h| { + const tile_max = if (w == 1) local_max[h] else t.readReduction(key_warps, row0.add(h * 8), true); + m[h] = if (single) tile_max else previous[h].maximum(tile_max); + // Empty/fully masked row tiles have m=-inf and denominator 0. + alpha[h] = if (single) zero else b.exp2Approx(previous[h].sub(m[h].maximum(-std.math.floatMax(f32)))); + } + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + var unique = b.openIf(t.warp.lt(key_warps)); + for (0..2) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } + unique.close(); + } + if (w == 1) { + for (&local_sum) |*value| inline for (.{ 2, 1 }) |step| { + value.* = value.*.add(b.shuffleXor(value.*, step)); + }; + } else { + var unique = b.openIf(t.warp.lt(key_warps)); + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + unique.close(); + } + b.cpAsyncWaitAll(); + t.sync(); + var next: [count]V = undefined; + for (0..2) |h| { + next[h] = m[h]; + const tile_sum = if (w == 1) local_sum[h] else t.readReduction(key_warps, row0.add(h * 8), false); + next[h + 2] = if (single) tile_sum else previous[h + 2].mul(alpha[h]).add(tile_sum); + } + var acc: [of][4]V = undefined; + for (0..of) |i| for (0..4) |j| { + acc[i][j] = if (single) zero else b.vectorExtract(previous[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); + }; + var packed_acc: [of]V = undefined; + for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); + var pv = b.openFor(0, n, 16, packed_acc); + { + const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + for (0..of) |i| { + const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); + var prev: [4]V = undefined; + for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); + acc[i] = mma(b, dt, pa, vb, prev); + packed_acc[i] = packAcc(b, acc[i]); + } + pv.yield(packed_acc); + } + @memcpy(next[4..], &pv.results); + if (!single) t.sync(); + return next; +} + pub const Merge = cute.Kernel(Config, .{ .name = "paged_attention_cute_indexed_grouped_merge", .inputs = &.{ "partials", "starts", "sink" }, @@ -472,67 +701,77 @@ pub const Merge = cute.Kernel(Config, .{ .run = merge, }); +// Lanes assigned to a feature own disjoint split partitions. Shuffles combine +// their softmax state, shortening the serial global-load chain at small batch. fn merge(b: *B, c: Config) cute.FinishError!void { + std.debug.assert(c.splits > 0 and c.splits <= 128 and (c.merge_parts == 4 or c.merge_parts == 8 or c.merge_parts == 16 or c.merge_parts == 32)); const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.tokens, c.kv_heads * c.group, c.splits, c.partialWidth() } } }, .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, }); - const tile = @min(@as(i64, 64), c.head_dim); - const dim_base = b.blockIdx().z.mul(tile); + const tid = b.threadIdx().x; + const cols = @divExact(32, c.merge_parts); + const part_lane = tid.rem(32).div(cols); + const dim = b.blockIdx().z.mul(4 * cols).add(tid.div(32).mul(cols)).add(tid.rem(cols)); const token = b.blockIdx().x; const head = b.blockIdx().y; - var inactive = b.openIf(token.ge(a.starts.get(.{c.batch}))); - { - var dims = b.openFor(b.threadIdx().x, tile, 64, .{}); - { - a.out.set(.{ token, head, dim_base.add(dims.iv) }, b.constant(c.dtype, 0)); - dims.close(); - } - inactive.close(); - } + const zero = b.constant(.f32, 0); + var inactive = b.openIf(token.ge(a.starts.get(.{c.batch})).bitAnd(part_lane.eq(0))); + a.out.set(.{ token, head, dim }, b.constant(c.dtype, 0)); + inactive.close(); var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); { - const zero = b.constant(.f32, 0); const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); - var maximum = b.openFor(b.threadIdx().x.rem(32), c.splits, 32, .{sink}); - { - maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + var maxima: [32]V = @splat(sink); + const groups = @divTrunc(c.splits + c.merge_parts - 1, c.merge_parts); + for (0..@as(usize, @intCast(groups))) |i| { + const part = part_lane.add(i * @as(usize, @intCast(c.merge_parts))); + maxima[i] = b.select(part.lt(c.splits), a.partials.get(.{ token, head, part.minimum(c.splits - 1), c.head_dim }), sink); } - var maximum_value = maximum.results[0]; - inline for (.{ 16, 8, 4, 2, 1 }) |step| maximum_value = maximum_value.maximum(b.shuffleXor(maximum_value, step)); - const m = maximum_value.maximum(-std.math.floatMax(f32)); - var dims = b.openFor(b.threadIdx().x, tile, 64, .{}); + var max_stride: usize = 1; + while (max_stride < 32) : (max_stride *= 2) { + var i: usize = 0; + while (i < 32) : (i += 2 * max_stride) maxima[i] = maxima[i].maximum(maxima[i + max_stride]); + } + var m = maxima[0].maximum(sink); + var step = cols; + while (step < 32) : (step *= 2) m = m.maximum(b.shuffleXor(m, @intCast(step))); + m = m.maximum(-std.math.floatMax(f32)); + const chunk = @min(@as(i64, 32), @divTrunc(c.splits + c.merge_parts - 1, c.merge_parts)); + const sink_weight = if (c.sink) b.select(part_lane.eq(0), b.exp2Approx(sink.sub(m)), zero) else zero; + var sum = b.openFor(0, c.splits, chunk * c.merge_parts, .{ zero, sink_weight }); { - // Expose independent loads and a balanced sum within each chunk. - // A bounded chunk avoids register spills for long selections. - const chunk = @min(@as(i64, 16), c.splits); - var sum = b.openFor(0, c.splits, chunk, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); - { - var values: [16]V = @splat(zero); - var denominators: [16]V = @splat(zero); - for (0..@as(usize, @intCast(chunk))) |i| { - const part = sum.iv.add(i); - const valid_part = part.lt(c.splits); - const safe_part = part.minimum(c.splits - 1); - const weight = b.select(valid_part, b.exp2Approx(a.partials.get(.{ safe_part, token, head, c.head_dim }).sub(m)), zero); - values[i] = weight.mul(a.partials.get(.{ safe_part, token, head, dim_base.add(dims.iv) })); - denominators[i] = weight.mul(a.partials.get(.{ safe_part, token, head, c.head_dim + 1 })); - } - var stride: usize = 1; - while (stride < 16) : (stride *= 2) { - var i: usize = 0; - while (i < 16) : (i += 2 * stride) { - values[i] = values[i].add(values[i + stride]); - denominators[i] = denominators[i].add(denominators[i + stride]); - } - } - sum.yield(.{ sum.carried[0].add(values[0]), sum.carried[1].add(denominators[0]) }); + var values: [32]V = @splat(zero); + var denominators: [32]V = @splat(zero); + for (0..@as(usize, @intCast(chunk))) |i| { + const part = sum.iv.add(part_lane).add(i * @as(usize, @intCast(c.merge_parts))); + const safe_part = part.minimum(c.splits - 1); + const weight = b.select(part.lt(c.splits), b.exp2Approx(a.partials.get(.{ token, head, safe_part, c.head_dim }).sub(m)), zero); + values[i] = weight.mul(a.partials.get(.{ token, head, safe_part, dim })); + denominators[i] = weight.mul(a.partials.get(.{ token, head, safe_part, c.head_dim + 1 })); } - a.out.set(.{ token, head, dim_base.add(dims.iv) }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); - dims.close(); + var stride: usize = 1; + while (stride < 32) : (stride *= 2) { + var i: usize = 0; + while (i < 32) : (i += 2 * stride) { + values[i] = values[i].add(values[i + stride]); + denominators[i] = denominators[i].add(denominators[i + stride]); + } + } + sum.yield(.{ sum.carried[0].add(values[0]), sum.carried[1].add(denominators[0]) }); } + var numerator = sum.results[0]; + var denominator = sum.results[1]; + step = cols; + while (step < 32) : (step *= 2) { + numerator = numerator.add(b.shuffleXor(numerator, @intCast(step))); + denominator = denominator.add(b.shuffleXor(denominator, @intCast(step))); + } + var leader = b.openIf(part_lane.eq(0)); + a.out.set(.{ token, head, dim }, numerator.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)).to(c.dtype)); + leader.close(); valid.close(); } } From 357781354e90ac2f1dbf9b55a6073fad9a39e96e Mon Sep 17 00:00:00 2001 From: dmehala Date: Mon, 16 Nov 2026 00:15:31 +0100 Subject: [PATCH 3/3] wip --- .../paged_attention/cute/benchmark.zig | 53 ++- .../paged_attention/cute/correctness_test.zig | 28 +- zml/kernels/paged_attention/cute/cute.zig | 57 +++- zml/kernels/paged_attention/cute/fixtures.zig | 6 +- .../paged_attention/cute/sparse_attention.zig | 323 +++++++++++++----- 5 files changed, 369 insertions(+), 98 deletions(-) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 2ccb2955..923d0ee3 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -8,6 +8,7 @@ const Inputs = fixtures.Inputs; const Args = struct { profile_dir: ?[]const u8 = null, paired: bool = false, + tuning: zml.paged_attention.cute.SparseTuning = .{}, sparse: bool = false, stress: bool = false, all_backends: bool = false, @@ -31,6 +32,7 @@ pub fn main(init: std.process.Init) !void { }); std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload, reference validation and output destruction)\n", .{platform.devices.len}); + std.debug.print("TUNING {any}\n", .{args.tuning}); const suite: []const cases.Case = if (args.stress) &cases.sparseStressBenchmarks else if (args.sparse) &cases.sparseBenchmarks else &cases.benchmarks; var failures: usize = 0; var selected: usize = 0; @@ -83,6 +85,20 @@ fn parseArgs(init: std.process.Init) !Args { } else if (std.mem.eql(u8, arg, "--stress")) { result.sparse = true; result.stress = true; + } else if (std.mem.startsWith(u8, arg, "--tune-head-rows=")) { + result.tuning.headRows = try std.fmt.parseInt(i64, arg["--tune-head-rows=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-warps=")) { + result.tuning.warps = try std.fmt.parseInt(i64, arg["--tune-warps=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-tile=")) { + result.tuning.kvTile = try std.fmt.parseInt(i64, arg["--tune-tile=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-splits=")) { + result.tuning.splits = try std.fmt.parseInt(i64, arg["--tune-splits=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-merge-lanes=")) { + result.tuning.mergeLanes = try std.fmt.parseInt(i64, arg["--tune-merge-lanes=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-merge-vector=")) { + result.tuning.mergeVector = try std.fmt.parseInt(i64, arg["--tune-merge-vector=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--tune-merge-parts=")) { + result.tuning.mergeParts = try std.fmt.parseInt(i64, arg["--tune-merge-parts=".len..], 10); } else if (std.mem.startsWith(u8, arg, "--backend=")) { const value = arg["--backend=".len..]; result.all_backends = std.mem.eql(u8, value, "all"); @@ -100,11 +116,18 @@ fn parseArgs(init: std.process.Init) !Args { result.samples = try std.fmt.parseInt(usize, arg["--samples=".len..], 10); if (result.samples == 0) return error.InvalidSamples; } else if (std.mem.eql(u8, arg, "--help")) { - std.debug.print("paged attention benchmark [--sparse|--stress] [--paired] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); + std.debug.print("paged attention benchmark [--sparse|--stress] [--paired] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH] [--tune-head-rows=N --tune-warps=N --tune-tile=N --tune-splits=N --tune-merge-parts=N --tune-merge-vector=N --tune-merge-lanes=N ]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); std.process.exit(0); } else return error.UnknownArgument; } - if (result.paired and (result.stress or result.backend != null or result.profile_dir != null)) return error.InvalidPairedOptions; + if (result.tuning.headRows) |v| if (v != 8 and v != 16) return error.InvalidTuning; + if (result.tuning.splits) |v| if (v < 1 or v > 128) return error.InvalidTuning; + if (result.tuning.mergeLanes) |v| if (std.mem.indexOfScalar(i64, &.{ 8, 16, 32 }, v) == null) return error.InvalidTuning; + if (result.tuning.mergeVector) |v| if (std.mem.indexOfScalar(i64, &.{ 0, 4, 8, 16 }, v) == null) return error.InvalidTuning; + if (result.tuning.warps) |v| if (std.mem.indexOfScalar(i64, &.{ 1, 2, 4, 8 }, v) == null) return error.InvalidTuning; + if (result.tuning.kvTile) |v| if (std.mem.indexOfScalar(i64, &.{ 16, 32, 64, 128 }, v) == null) return error.InvalidTuning; + if (result.tuning.mergeParts) |v| if (std.mem.indexOfScalar(i64, &.{ 4, 8, 16, 32 }, v) == null) return error.InvalidTuning; + if (result.paired and (result.stress or result.backend != null)) return error.InvalidPairedOptions; return result; } @@ -119,7 +142,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend, args.tuning }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -180,18 +203,19 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtures.Backend) zml.Tensor { +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtures.Backend, tuning: zml.paged_attention.cute.SparseTuning) zml.Tensor { const Call = struct { inputs: Inputs, case: cases.Case, backend: fixtures.Backend, + tuning: zml.paged_attention.cute.SparseTuning, fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forwardBackend(self.inputs, self.case, self.backend); + return fixtures.forwardBackendTuned(self.inputs, self.case, self.backend, self.tuning); } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c, .backend = backend }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c, .backend = backend, .tuning = tuning }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, @@ -238,7 +262,9 @@ fn runPaired(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, me for (&samples) |*v| v.* = try arena.allocator().alloc(f64, args.samples); for (backends, 0..) |backend, i| { const start: std.Io.Timestamp = .now(io, .awake); - executables[i] = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend }, .{ .program_name = "paired_attention", .meshes = &.{mesh} }); + const programName = try std.fmt.allocPrint(a, "paired_attention_{s}", .{@tagName(backend)}); + defer a.free(programName); + executables[i] = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend, args.tuning }, .{ .program_name = programName, .meshes = &.{mesh} }); compiled += 1; compile_us[i] = elapsedUs(start, io); runners[i] = try executables[i].runner(a); @@ -250,7 +276,19 @@ fn runPaired(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, me defer host.free(a); errors[i] = if (args.dtype == .f16) try reference.check(a, global_case, fixture, host.items(f16)) else try reference.check(a, global_case, fixture, host.items(zml.floats.BFloat16)); } + const session = try std.fmt.allocPrint(a, "{s}-{d}-paired", .{ @tagName(args.dtype), index }); + defer a.free(session); + var profiler = if (args.profile_dir) |path| try platform.profiler(a, io, .{ + .repository_path = path, + .session_id = session, + .host_tracer_level = 0, + .device_tracer_level = 1, + .python_tracer_level = 0, + .raise_error_on_start_failure = true, + }) else null; + defer if (profiler) |*p| p.deinit(); for (0..args.warmup + args.samples) |round| { + if (round == args.warmup) if (profiler) |*p| try p.start(); for (0..backends.len) |rank| { const i = (rank + round) % backends.len; var output: zml.Buffer = undefined; @@ -261,6 +299,7 @@ fn runPaired(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, me output.deinit(); } } + if (profiler) |*p| _ = try p.stop(); const deltas = try arena.allocator().alloc(f64, args.samples); const ratios = try arena.allocator().alloc(f64, args.samples); const cute_index = @backingInt(fixtures.Backend.cute); diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index b1bf3b8b..b7894fb8 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -69,6 +69,28 @@ test "indexed long selections handle sinks tails and inactive capacity" { }; } +test "indexed eight-head tiles handle tails and packed capacity" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + const suite = [_]cases.Case{ + .{ .name = "sparse_eight_head_mixed_tail", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 19, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .kv_len = 2049, .mask = .indices, .topk = 513, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_eight_head_small_tail", .head_dim = 64, .batch = 4, .group = 5, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .mask = .indices, .topk = 65, .sink = true, .shuffle_pages = true }, + .{ .name = "sparse_eight_head_empty_capacity", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 16, .inactive = 1, .token_capacity = 6, .mask = .indices, .topk = 513, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, + }; + for ([_]zml.DataType{ .bf16, .f16 }) |dtype| for (suite) |c| { + std.debug.print("CASE name={s} backend=cute compute_dtype={s}\n", .{ c.name, @tagName(dtype) }); + const tuning: zml.paged_attention.cute.SparseTuning = .{ + .headRows = 8, + .warps = 8, + .kvTile = 32, + .splits = if (c.head_dim == 64) 3 else null, + .mergeLanes = if (c.head_dim == 64) 8 else 32, + .mergeVector = if (c.head_dim == 64) 4 else 8, + }; + try runCorrectnessTuned(std.testing.allocator, std.testing.io, platform, c, .cute, dtype, tuning); + }; +} + fn runSparseCorrectness(dtype: zml.DataType) !void { const platform = zml.testing.env(); if (platform.target != .cuda) return error.CudaRequired; @@ -88,8 +110,12 @@ fn runSparseCorrectness(dtype: zml.DataType) !void { } fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, backend: fixtures.Backend, dtype: zml.DataType) !void { + return runCorrectnessTuned(a, io, platform, c, backend, dtype, .{}); +} + +fn runCorrectnessTuned(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, backend: fixtures.Backend, dtype: zml.DataType, tuning: zml.paged_attention.cute.SparseTuning) !void { const inputs = Inputs.initDType(c, dtype); - const exe = try platform.compileFn(a, io, fixtures.forwardBackend, .{ inputs, c, backend }, .{ .program_name = "paged_attention_harness" }); + const exe = try platform.compileFn(a, io, fixtures.forwardBackendTuned, .{ inputs, c, backend, tuning }, .{ .program_name = "paged_attention_harness" }); defer exe.deinit(); var runner = try exe.runner(a); diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 8cd2b659..733c9d68 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -82,7 +82,7 @@ fn sparseIndices(q: zml.Tensor, indices: zml.Tensor) zml.Tensor { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { validateInputs(q, k, v); if (opts.mask == .indices) - return pagedSparseAttention(parameters, q, k, v, opts); + return pagedSparseAttention(parameters, q, k, v, opts, .{}); if (sm10x.isAvailable(zml.Compiler.current().platform)) return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform)) @@ -134,7 +134,25 @@ pub fn pagedAttentionGeneric(parameters: Parameters, q: zml.Tensor, k: zml.Tenso }).out.transpose(q.shape()); } -fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { +// Overrides let the kernel benchmark evaluate launch geometry without changing +// the production policy or either comparison backend. +pub const SparseTuning = struct { + headRows: ?i64 = null, + warps: ?i64 = null, + kvTile: ?i64 = null, + splits: ?i64 = null, + mergeParts: ?i64 = null, + mergeVector: ?i64 = null, + mergeLanes: ?i64 = null, +}; + +pub fn pagedAttentionTuned(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions, tuning: SparseTuning) zml.Tensor { + validateInputs(q, k, v); + stdx.debug.assert(opts.mask == .indices, "Sparse tuning requires indexed attention", .{}); + return pagedSparseAttention(parameters, q, k, v, opts, tuning); +} + +fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions, tuning: SparseTuning) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); @@ -160,7 +178,36 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: .sink = opts.sink != null, .decode = all_decode and parameters.options_.max_seqlen_q == 1, }; + const platform = zml.Compiler.current().platform; + const cc = zml.platform.cuda.computeCapability(platform).?; + const narrowDecode = cfg.decode and cfg.tokens == 1 and cfg.batch == 1 and + ((cfg.head_dim == 64 and cfg.group <= 8 and cfg.topk > 128) or + (cfg.head_dim == 512 and cfg.group <= 16 and cfg.topk > 512)); + // Underfilled GB300 grids benefit from eight real query rows per CTA. + // Vector merges amortize each partition's softmax state across features. + const narrowWork = cfg.kv_heads * @divTrunc(cfg.group + 7, 8) * @divTrunc(cfg.topk + 31, 32); + if (cc.major == 10 and cc.minor == 3 and narrowDecode and narrowWork <= @as(i64, @intCast(multiprocessorCount(platform)))) { + cfg.head_rows = 8; + cfg.warps = 8; + cfg.kv_tile = 32; + cfg.merge_vector = if (cfg.head_dim == 512) 8 else 4; + cfg.merge_lanes = if (cfg.head_dim == 512) 32 else 8; + } + if (tuning.headRows) |rows| { + stdx.debug.assert(rows == 8 or rows == 16, "Invalid sparse head tile", .{}); + cfg.head_rows = rows; + } + if (tuning.warps) |warps| cfg.warps = warps; + if (tuning.kvTile) |tile| cfg.kv_tile = tile; cfg.merge_parts = if (cfg.head_dim >= 128 and cfg.head_dim <= 256 and cfg.tokens * cfg.kv_heads * cfg.group <= 16) 16 else 8; + if (tuning.mergeParts) |parts| cfg.merge_parts = parts; + if (tuning.mergeLanes) |lanes| cfg.merge_lanes = lanes; + stdx.debug.assert(cfg.merge_lanes == 8 or cfg.merge_lanes == 16 or cfg.merge_lanes == 32, "Invalid sparse merge lanes", .{}); + if (tuning.mergeVector) |features| { + stdx.debug.assert(features == 0 or ((features == 4 or features == 8 or features == 16) and cfg.head_dim >= @divExact(128, cfg.merge_lanes) * features), "Invalid sparse merge vector", .{}); + cfg.merge_vector = features; + } + stdx.debug.assert(@mod(cfg.head_dim, cfg.mergeFeatures()) == 0, "Sparse merge tile must divide the head dimension", .{}); const work = @max(1, cfg.tokens * cfg.kv_heads * cfg.headTiles()); // Short selections avoid a merge; longer decode selections expose one // selected tile per partition until the device has enough parallel work. @@ -169,6 +216,10 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: var desired: i64 = 1; while (desired * work < target) desired *= 2; cfg.splits = if (short_selection) 1 else if (all_decode) @max(1, @min(128, @min(tiles, desired))) else if (cfg.head_dim <= 128) 1 else @max(1, @min(32, @min(@divTrunc(@divTrunc(target, 8) + work - 1, work), @divTrunc(cfg.topk, 128)))); + if (tuning.splits) |splits| { + stdx.debug.assert(splits > 0 and splits <= tiles and splits <= 128, "Invalid sparse split count", .{}); + cfg.splits = splits; + } const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.scalar(0, .f32); if (cfg.group == 1 and cfg.topk <= 8) return sparse.Tiny.call(.{ .q = q_, @@ -200,7 +251,7 @@ fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: }).out; const result = if (cfg.splits > 1) sparse.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divTrunc(cfg.head_dim, @divExact(128, cfg.merge_parts))) }, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divExact(cfg.head_dim, cfg.mergeFeatures())) }, .block = .{ 128, 1, 1 }, }).out else out; return result.transpose(q.shape()); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 435ec525..15b6eb09 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -140,6 +140,10 @@ pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { } pub fn forwardBackend(i: Inputs, c: cases.Case, backend: Backend) zml.Tensor { + return forwardBackendTuned(i, c, backend, .{}); +} + +pub fn forwardBackendTuned(i: Inputs, c: cases.Case, backend: Backend, tuning: zml.paged_attention.cute.SparseTuning) zml.Tensor { const parameters: zml.paged_attention.cute.Parameters = .{ .block_table = i.table, .seq_lens = i.lengths, @@ -152,7 +156,7 @@ pub fn forwardBackend(i: Inputs, c: cases.Case, backend: Backend) zml.Tensor { }, }; return switch (backend) { - .cute => zml.paged_attention.cute.pagedAttention(parameters, i.q, i.k, i.v, i.options(c)), + .cute => if (c.mask == .indices) zml.paged_attention.cute.pagedAttentionTuned(parameters, i.q, i.k, i.v, i.options(c), tuning) else zml.paged_attention.cute.pagedAttention(parameters, i.q, i.k, i.v, i.options(c)), .cute_generic => zml.paged_attention.cute.pagedAttentionGeneric(parameters, i.q, i.k, i.v, i.options(c)), .triton => zml.paged_attention.triton.paged.pagedAttention(.{ .block_table = i.table, diff --git a/zml/kernels/paged_attention/cute/sparse_attention.zig b/zml/kernels/paged_attention/cute/sparse_attention.zig index f9021041..c19d7df6 100644 --- a/zml/kernels/paged_attention/cute/sparse_attention.zig +++ b/zml/kernels/paged_attention/cute/sparse_attention.zig @@ -25,15 +25,17 @@ pub const Config = struct { scale: f32, sink: bool, decode: bool = false, + head_rows: i64 = 16, merge_parts: i64 = 8, + merge_vector: i64 = 0, + merge_lanes: i64 = 32, splits: i64 = 1, pub fn rows(c: Config) i64 { - _ = c; - return 16; + return c.head_rows; } pub fn headTiles(c: Config) i64 { - return @divTrunc(c.group + c.rows() - 1, c.rows()); + return @divTrunc(c.group + c.head_rows - 1, c.head_rows); } fn kvBase(c: Config) i64 { return c.rows() * c.head_dim * 2; @@ -42,6 +44,9 @@ pub const Config = struct { // Align each split to a 32-byte sector, including max/denominator. return c.head_dim + 8; } + pub fn mergeFeatures(c: Config) i64 { + return if (c.merge_vector > 0) @divExact(128, c.merge_lanes) * c.merge_vector else @divExact(128, c.merge_parts); + } fn pBase(c: Config) i64 { return c.kvBase() + c.kv_tile * c.head_dim * 2; } @@ -66,7 +71,7 @@ pub const Kernel = cute.Kernel(Config, .{ .run = run, }); -// Small MHA selections use SIMT: four warps evaluate independent selected +// Small MHA selections use SIMT: warps evaluate independent selected // slots, then reduce their weighted values within one CTA and one launch. pub const Tiny = cute.Kernel(Config, .{ .name = "paged_attention_cute_indexed_tiny", @@ -304,7 +309,7 @@ const TileContext = struct { } else t.b.syncThreads(); } - // Resolve each selected slot once, then share its physical page across + // Resolve each selected slot once, then share its byte offset across // every head, feature vector, and both K/V loads. Invalid logical indices // never dereference the page table; cp.async zero-fills their K/V rows. fn gather(t: TileContext, key_base: V) void { @@ -313,8 +318,7 @@ const TileContext = struct { var selected = b.openFor(t.tid, c.kv_tile, c.warps * 32, .{}); { const address = t.shared.add(c.indexBase()).add(selected.iv.mul(8)); - b.storeWords(1, address, .shared, .{b.constant(.i32, -1)}); - b.storeWords(1, address.add(4), .shared, .{b.constant(.i32, 0)}); + b.emitVoid(llvm.store(b.ctx, b.constant(.i64, -1).inner, b.addressPtr(address, .shared).inner, .{ .alignment = 8 }, b.loc())); const slot = key_base.add(selected.iv); var in_range = b.openIf(slot.lt(c.topk)); { @@ -322,8 +326,8 @@ const TileContext = struct { var valid = b.openIf(key.ge(0).bitAnd(key.lt(t.klen))); { const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(key.div(c.page_size))); - b.storeWords(1, address, .shared, .{page}); - b.storeWords(1, address.add(4), .shared, .{key.rem(c.page_size)}); + const resolvedOffset = page.to(.i64).mul(c.page_size).add(key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim * 2); + b.emitVoid(llvm.store(b.ctx, resolvedOffset.inner, b.addressPtr(address, .shared).inner, .{ .alignment = 8 }, b.loc())); valid.close(); } in_range.close(); @@ -333,8 +337,15 @@ const TileContext = struct { t.sync(); } + // Preserve offsets beyond 2 GiB in large paged caches. + fn byteOffset(t: TileContext, row: V) V { + const b = t.b; + const address = t.shared.add(t.c.indexBase()).add(row.mul(8)); + return b.emit(llvm.load(b.ctx, b.addressPtr(address, .shared).inner, cute.DType.i64.toMlir(b.ctx), .{ .alignment = 8 }, b.loc())); + } + fn validSlot(t: TileContext, row: V) V { - return t.b.loadWords(1, t.shared.add(t.c.indexBase()).add(row.mul(8)), .shared, .{})[0].ge(0); + return t.byteOffset(row).ge(0); } fn loadKV(t: TileContext, source: V) void { @@ -357,14 +368,10 @@ const TileContext = struct { const c = t.c; const row = iv.div(@divExact(c.head_dim, 8)); const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); - const address = t.shared.add(c.indexBase()).add(row.mul(8)); - const mapping = b.loadWords(2, address, .shared, .{}); - const page = mapping[0]; - const within = mapping[1]; - const valid = page.ge(0); - const safe_page = b.select(valid, page, b.constant(.i32, 0)); - const offset = safe_page.to(.i64).mul(c.page_size).add(within.to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); - b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + const offset = t.byteOffset(row); + const valid = offset.ge(0); + const safeOffset = b.select(valid, offset, b.constant(.i64, 0)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(safeOffset).add(col.to(.i64).mul(2)), valid); } fn loadQuery(t: TileContext) void { @@ -387,12 +394,21 @@ const TileContext = struct { const row = iv.div(@divExact(c.head_dim, 8)); const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); const packed_row = t.first_row.add(row); - const valid = packed_row.lt(c.group); + const valid = packed_row.lt(c.group).bitAnd(row.lt(c.head_rows)); const safe_row = t.b.select(valid, packed_row, t.b.constant(.i32, 0)); const offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(safe_row.to(.i64)).mul(c.head_dim).add(col.to(.i64)); t.b.cpAsync16(t.shared.add(swizzle(row, col, c.head_dim)), t.q.add(offset.mul(2)), valid); } + fn loadRows(t: TileContext, base: V, rowBase: V, colBase: V, width: i64) [4]V { + if (t.c.head_rows == 8) { + const pair = matrixLoad(t.b, 2, false, base.add(swizzle(rowBase.add(t.lane.rem(8)), colBase.add(t.lane.div(8).rem(2).mul(8)), width))); + const zero = t.b.constant(.i32, 0); + return .{ pair[0], zero, pair[1], zero }; + } + return matrixLoad(t.b, 4, false, base.add(swizzle(rowBase.add(t.lane.rem(16)), colBase.add(t.lane.div(16).mul(8)), width))); + } + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { const b = t.b; var v = value; @@ -417,6 +433,18 @@ const TileContext = struct { return result; } + fn storePair(t: TileContext, row: V, dim: V, first: V, second: V, comptime dt: cute.DType) void { + const c = t.c; + var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); + if (c.splits > 1) { + offset = offset.mul(c.splits).add(t.split.to(.i64)).mul(c.partialWidth()).add(dim.to(.i64)); + t.b.storeWords(2, t.out.add(offset.mul(4)), .global, .{ first.bitCast(.i32), second.bitCast(.i32) }); + } else { + offset = offset.mul(c.head_dim).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(offset.mul(2)), .global, .{t.b.packHalf2(first, second, dt)}); + } + } + fn store(t: TileContext, row: V, dim: V, value: V) void { const c = t.c; var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); @@ -445,7 +473,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u }); if (c.sharedBytes() > 48 * 1024) b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", .int(b.ctx, .i32, c.sharedBytes()))}))); const token = b.blockIdx().x; - const first_row = b.blockIdx().y.rem(c.headTiles()).mul(c.rows()); + const first_row = b.blockIdx().y.rem(c.headTiles()).mul(c.head_rows); const head = b.blockIdx().y.div(c.headTiles()); const starts = b.ptrToInt(a.starts, .i64); const active_tokens = loadI32(b, starts, b.constant(.i32, c.batch)); @@ -458,7 +486,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u const width = if (c.splits > 1) c.partialWidth() else c.head_dim; const row = first_row.add(dims.iv.div(width)); const dim = dims.iv.rem(width); - var valid = b.openIf(row.lt(c.group)); + var valid = b.openIf(row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows)))); { var offset = token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(c.group).add(row.to(.i64)); if (c.splits > 1) { @@ -526,9 +554,9 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u break :blk loop.results; }; if (c.splits > tiles) b.cpAsyncWaitAll(); - for (0..2) |h| { + for (0..@as(usize, @intCast(@divExact(c.head_rows, 8)))) |h| { const row = first_row.add(row_base).add(row0).add(h * 8); - var valid = b.openIf(row.lt(c.group)); + var valid = b.openIf(row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows)))); { const m = result[h]; const l = result[h + 2]; @@ -540,10 +568,14 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u const denominator = l.mul(weight).add(b.exp2Approx(sink.sub(maximum))); norm = weight.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)); } - for (0..of) |i| for (0..2) |j| { - const value = b.vectorExtract(result[4 + i], @intCast(h * 2 + j), .f32); - t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); - }; + for (0..of) |i| { + var pair: [2]V = undefined; + for (&pair, 0..) |*value, j| { + const acc = b.vectorExtract(result[4 + i], @intCast(h * 2 + j), .f32); + value.* = if (c.splits > 1) acc else acc.mul(norm); + } + t.storePair(row, dim_part.add(i * 8).add(col0), pair[0], pair[1], dt); + } if (c.splits > 1) { var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); { @@ -574,6 +606,7 @@ fn advanceTile(t: TileContext, comptime d: usize, comptime w: usize, comptime n: const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); const nf = n / key_warps / 8; + const halves: usize = @intCast(@divExact(c.head_rows, 8)); const of = d / w / 8; const count = 4 + of; t.gather(key_base); @@ -584,91 +617,98 @@ fn advanceTile(t: TileContext, comptime d: usize, comptime w: usize, comptime n: // Keep only MMA operands live in registers; Q remains in shared // memory so D512 does not consume the entire per-thread register file. for (0..d / feature_parts / 16) |ki| { - const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), feature_part.add(ki * 16).add(t.lane.div(16).mul(8)), d))); + const qa = t.loadRows(t.shared, row_base, feature_part.add(ki * 16), d); for (0..nf) |i| { const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), feature_part.add(ki * 16).add(t.lane.div(8).rem(2).mul(8)), d))); scores[i] = mma(b, dt, qa, kb, scores[i]); } } if (feature_parts > 1) { - // Store each MMA register as a contiguous warp vector; row-major - // FP32 score tiles alias shared-memory banks across query rows. - // The max-reduction barrier protects these reads before reuse as P. + // Publishing partial dots proves every warp has finished reading K. + // Reuse its storage for V while the score warps combine the dots. const bank = t.warp.div(key_warps); - for (0..nf) |i| for (0..4) |j| { - const register = t.warp.rem(key_warps).mul(nf * 128).add(i * 128 + j * 32).add(t.lane); - const offset = bank.mul(16 * n).add(register).mul(4); + for (0..nf) |i| for (0..halves * 2) |j| { + const register = t.warp.rem(key_warps).mul(@as(i64, nf) * c.head_rows * 8).add(i * @as(usize, @intCast(c.head_rows * 8)) + j * 32).add(t.lane); + const offset = bank.mul(c.head_rows * @as(i64, n)).add(register).mul(4); b.storeWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{scores[i][j].bitCast(.i32)}); }; t.sync(); - for (0..nf) |i| for (0..4) |j| { - const register = t.warp.rem(key_warps).mul(nf * 128).add(i * 128 + j * 32).add(t.lane); - for (1..feature_parts) |peer| { - const offset = bank.add(peer).rem(feature_parts).mul(16 * n).add(register).mul(4); - scores[i][j] = scores[i][j].add(b.loadWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{})[0].bitCast(.f32)); - } - }; + t.loadKV(t.v); } - var local_max: [2]V = @splat(neg_inf); - for (0..nf) |i| { - for (0..4) |j| { + const computesScores = if (feature_parts > 1) t.warp.lt(key_warps) else b.constant(.i1, 1); + var scoreStage = b.openIfElse(computesScores, @as([nf * 4 + 2]V, @splat(zero))); + { + if (feature_parts > 1) { + // Only the score warps combine dots and evaluate softmax. The + // remaining warps own output features and consume the shared P. + for (0..nf) |i| for (0..halves * 2) |j| { + const register = t.warp.rem(key_warps).mul(@as(i64, nf) * c.head_rows * 8).add(i * @as(usize, @intCast(c.head_rows * 8)) + j * 32).add(t.lane); + for (1..feature_parts) |peer| { + const offset = b.constant(.i32, peer * @as(usize, @intCast(c.head_rows)) * n).add(register).mul(4); + scores[i][j] = scores[i][j].add(b.loadWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{})[0].bitCast(.f32)); + } + }; + } + var localMax: [2]V = @splat(neg_inf); + for (0..nf) |i| for (0..4) |j| { const row = first_row.add(row_base).add(row0).add((j / 2) * 8); const slot = key_part.add(i * 8).add(col0).add(j % 2); - const valid = row.lt(c.group).bitAnd(t.validSlot(slot)); + const valid = row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows))).bitAnd(t.validSlot(slot)); scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); - local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); - } - } - if (w == 1) { - for (&local_max) |*value| inline for (.{ 2, 1 }) |step| { - value.* = value.*.maximum(b.shuffleXor(value.*, step)); + localMax[j / 2] = localMax[j / 2].maximum(scores[i][j]); }; - } else { - var unique = b.openIf(t.warp.lt(key_warps)); - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); - unique.close(); - t.sync(); + if (w == 1) { + for (&localMax) |*value| inline for (.{ 2, 1 }) |step| { + value.* = value.*.maximum(b.shuffleXor(value.*, step)); + }; + } else for (0..halves) |h| t.publishReduction(row0.add(h * 8), localMax[h], true); + var result: [nf * 4 + 2]V = undefined; + @memcpy(result[0 .. nf * 4], &flatten(nf, scores)); + @memcpy(result[nf * 4 ..], &localMax); + scoreStage.yieldThen(result); } - // All K reads have completed. Reuse its allocation for V, and - // overlap these global loads with softmax and probability stores. - t.loadKV(t.v); - var m: [2]V = undefined; - var alpha: [2]V = undefined; + scoreStage.yieldElse(@as([nf * 4 + 2]V, @splat(zero))); + for (0..nf) |i| scores[i] = scoreStage.results[i * 4 ..][0..4].*; + const local_max = scoreStage.results[nf * 4 ..][0..2].*; + if (w > 1) t.sync(); + if (feature_parts == 1) t.loadKV(t.v); + var m: [2]V = @splat(neg_inf); + var alpha: [2]V = @splat(zero); var local_sum: [2]V = @splat(zero); - for (0..2) |h| { + for (0..halves) |h| { const tile_max = if (w == 1) local_max[h] else t.readReduction(key_warps, row0.add(h * 8), true); m[h] = if (single) tile_max else previous[h].maximum(tile_max); // Empty/fully masked row tiles have m=-inf and denominator 0. alpha[h] = if (single) zero else b.exp2Approx(previous[h].sub(m[h].maximum(-std.math.floatMax(f32)))); } - for (0..nf) |i| { - for (0..4) |j| { - scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); - local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + var probabilityStage = b.openIfElse(computesScores, @as([2]V, @splat(zero))); + { + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = if (j / 2 < halves) b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))) else zero; + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + for (0..halves) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } } - var unique = b.openIf(t.warp.lt(key_warps)); - for (0..2) |h| { - const row = row_base.add(row0).add(h * 8); - const col = key_part.add(i * 8).add(col0); - b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); - } - unique.close(); - } - if (w == 1) { - for (&local_sum) |*value| inline for (.{ 2, 1 }) |step| { - value.* = value.*.add(b.shuffleXor(value.*, step)); - }; - } else { - var unique = b.openIf(t.warp.lt(key_warps)); - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); - unique.close(); + if (w == 1) { + for (&local_sum) |*value| inline for (.{ 2, 1 }) |step| { + value.* = value.*.add(b.shuffleXor(value.*, step)); + }; + } else for (0..halves) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + probabilityStage.yieldThen(local_sum); } + probabilityStage.yieldElse(@as([2]V, @splat(zero))); + local_sum = probabilityStage.results; b.cpAsyncWaitAll(); t.sync(); var next: [count]V = undefined; for (0..2) |h| { next[h] = m[h]; - const tile_sum = if (w == 1) local_sum[h] else t.readReduction(key_warps, row0.add(h * 8), false); + const tile_sum = if (h >= halves) zero else if (w == 1) local_sum[h] else t.readReduction(key_warps, row0.add(h * 8), false); next[h + 2] = if (single) tile_sum else previous[h + 2].mul(alpha[h]).add(tile_sum); } var acc: [of][4]V = undefined; @@ -679,7 +719,7 @@ fn advanceTile(t: TileContext, comptime d: usize, comptime w: usize, comptime n: for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); var pv = b.openFor(0, n, 16, packed_acc); { - const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + const pa = t.loadRows(t.shared.add(c.pBase()), row_base, pv.iv, n); for (0..of) |i| { const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); var prev: [4]V = undefined; @@ -704,6 +744,115 @@ pub const Merge = cute.Kernel(Config, .{ // Lanes assigned to a feature own disjoint split partitions. Shuffles combine // their softmax state, shortening the serial global-load chain at small batch. fn merge(b: *B, c: Config) cute.FinishError!void { + if (c.merge_vector != 0) return switch (c.merge_vector) { + inline 4, 8, 16 => |features| switch (c.dtype) { + inline .bf16, .f16 => |dt| switch (c.merge_lanes) { + inline 8, 16, 32 => |parts| mergeVectorized(b, c, features, parts, dt), + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }; + return mergeScalar(b, c); +} + +// A lane owns a split and a contiguous feature vector. Reuse the score and +// weight for the entire vector, then reduce splits within each warp. +fn mergeVectorized(b: *B, c: Config, comptime features: usize, comptime parts: usize, comptime dt: cute.DType) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .ptr = cute.DType.f32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = dt }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + const lane = b.threadIdx().x.rem(32); + const warp = b.threadIdx().x.div(32); + const featureLanes = 32 / parts; + const featureLane = lane.rem(featureLanes); + const partLane = lane.div(featureLanes); + const dim = b.blockIdx().z.mul(4 * featureLanes * features).add(warp.mul(featureLanes * features)).add(featureLane.mul(features)); + const slots = 128 / parts; + const query = token.to(.i64).mul(c.kv_heads * c.group).add(head.to(.i64)); + const out = b.ptrToInt(a.out, .i64).add(query.mul(c.head_dim).add(dim.to(.i64)).mul(2)); + const activeTokens = loadI32(b, b.ptrToInt(a.starts, .i64), b.constant(.i32, c.batch)); + const zero = b.constant(.f32, 0); + const negInf = b.constant(.f32, -std.math.inf(f32)); + var inactive = b.openIf(token.ge(activeTokens).bitAnd(partLane.eq(0))); + if (features == 4) { + b.storeWords(2, out, .global, @splat(b.constant(.i32, 0))); + } else for (0..features / 8) |chunk| b.storeWords(4, out.add(chunk * 16), .global, @splat(b.constant(.i32, 0))); + inactive.close(); + var active = b.openIf(token.lt(activeTokens)); + { + const base = b.ptrToInt(a.partials, .i64).add(query.mul(c.splits * c.partialWidth() * 4)); + const sink = if (c.sink) b.loadWords(1, b.ptrToInt(a.sink, .i64).add(head.to(.i64).mul(4)), .global, .{ .invariant = true })[0].bitCast(.f32).mul(std.math.log2e) else negInf; + const count: usize = @intCast(@divTrunc(c.splits + parts - 1, parts)); + var maxima: [slots]V = @splat(negInf); + var sums: [slots]V = @splat(zero); + var addresses: [slots]V = undefined; + var valid: [slots]V = undefined; + for (0..count) |i| { + const part = partLane.add(i * parts); + valid[i] = part.lt(c.splits); + addresses[i] = base.add(part.minimum(c.splits - 1).to(.i64).mul(c.partialWidth() * 4)); + const stats = b.loadWords(2, addresses[i].add(c.head_dim * 4), .global, .{ .invariant = true }); + sums[i] = b.select(valid[i], stats[1].bitCast(.f32), zero); + maxima[i] = b.select(valid[i].bitAnd(sums[i].gt(0)), stats[0].bitCast(.f32), negInf); + } + var reducedMaxima = maxima; + var stride: usize = 1; + while (stride < slots) : (stride *= 2) { + var i: usize = 0; + while (i < slots) : (i += 2 * stride) reducedMaxima[i] = reducedMaxima[i].maximum(reducedMaxima[i + stride]); + } + var maximum = reducedMaxima[0].maximum(sink); + var step: usize = featureLanes; + while (step < 32) : (step *= 2) maximum = maximum.maximum(b.shuffleXor(maximum, @intCast(step))); + maximum = maximum.maximum(-std.math.floatMax(f32)); + var weights: [slots]V = @splat(zero); + var denominator = zero; + for (0..count) |i| { + weights[i] = b.exp2Approx(maxima[i].sub(maximum)); + denominator = denominator.add(weights[i].mul(sums[i])); + } + if (c.sink) denominator = denominator.add(b.select(partLane.eq(0), b.exp2Approx(sink.sub(maximum)), zero)); + step = featureLanes; + while (step < 32) : (step *= 2) denominator = denominator.add(b.shuffleXor(denominator, @intCast(step))); + const norm = b.select(denominator.gt(0), b.rcpApprox(denominator), zero); + var values: [slots][features]V = @splat(@splat(zero)); + for (0..count) |i| for (0..features / 4) |chunk| { + const words = b.loadWords(4, addresses[i].add(dim.to(.i64).mul(4)).add(chunk * 16), .global, .{ .invariant = true }); + for (words, 0..) |word, j| values[i][chunk * 4 + j] = b.select(valid[i], weights[i].mul(word.bitCast(.f32)), zero); + }; + var result: [features]V = undefined; + for (&result, 0..) |*value, j| { + stride = 1; + while (stride < slots) : (stride *= 2) { + var i: usize = 0; + while (i < slots) : (i += 2 * stride) values[i][j] = values[i][j].add(values[i + stride][j]); + } + value.* = values[0][j]; + step = featureLanes; + while (step < 32) : (step *= 2) value.* = value.*.add(b.shuffleXor(value.*, @intCast(step))); + value.* = value.*.mul(norm); + } + var writer = b.openIf(partLane.eq(0)); + if (features == 4) { + b.storeWords(2, out, .global, .{ b.packHalf2(result[0], result[1], dt), b.packHalf2(result[2], result[3], dt) }); + } else for (0..features / 8) |chunk| { + var words: [4]V = undefined; + for (&words, 0..) |*word, j| word.* = b.packHalf2(result[chunk * 8 + j * 2], result[chunk * 8 + j * 2 + 1], dt); + b.storeWords(4, out.add(chunk * 16), .global, words); + } + writer.close(); + active.close(); + } +} + +fn mergeScalar(b: *B, c: Config) cute.FinishError!void { std.debug.assert(c.splits > 0 and c.splits <= 128 and (c.merge_parts == 4 or c.merge_parts == 8 or c.merge_parts == 16 or c.merge_parts == 32)); const a = try b.declareArgs(.{ .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.tokens, c.kv_heads * c.group, c.splits, c.partialWidth() } } }, @@ -777,9 +926,10 @@ fn merge(b: *B, c: Config) cute.FinishError!void { } test "indexed grouped attention emits all supported head dimensions and dtypes" { - for ([_]i64{ 32, 64, 128, 256, 512 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| { + for ([_]i64{ 32, 64, 128, 256, 512 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| for ([_]i64{ 8, 16 }) |headRows| { const cfg: Config = .{ .head_dim = head_dim, + .head_rows = headRows, .tokens = 7, .batch = 3, .kv_heads = 2, @@ -792,11 +942,12 @@ test "indexed grouped attention emits all supported head dimensions and dtypes" .scale = 1 / @sqrt(@as(f32, @floatFromInt(head_dim))), .sink = false, }; - try std.testing.expectEqual(@as(i64, 2), cfg.headTiles()); + try std.testing.expectEqual(@as(i64, if (headRows == 8) 3 else 2), cfg.headTiles()); try std.testing.expect(cfg.sharedBytes() <= 99 * 1024); const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); defer std.testing.allocator.free(ir); try std.testing.expect(std.mem.indexOf(u8, ir, if (dtype == .bf16) "mma.sync.aligned.m16n8k16.row.col.f32.bf16" else "mma.sync.aligned.m16n8k16.row.col.f32.f16") != null); + try std.testing.expectEqual(headRows == 16, std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x4") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "cp.async") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); };