diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index 39d3fbbf..6329ae12 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -7,22 +7,11 @@ filegroup( srcs = [ "attention.zig", "cute.zig", - "sparse_attention.zig", "//zml/kernels/paged_attention/cute/sm10x", "//zml/kernels/paged_attention/cute/sm120", ], ) -zig_test( - name = "reference_test", - srcs = [ - "cases.zig", - "fixtures.zig", - ], - main = "reference.zig", - deps = ["//zml"], -) - zig_test( name = "correctness_test", srcs = [ @@ -47,7 +36,6 @@ zig_binary( srcs = [ "cases.zig", "fixtures.zig", - "reference.zig", ], main = "benchmark.zig", tags = ["manual"], diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 923d0ee3..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -2,220 +2,130 @@ const std = @import("std"); const zml = @import("zml"); const cases = @import("cases.zig"); const fixtures = @import("fixtures.zig"); -const reference = @import("reference.zig"); const Inputs = fixtures.Inputs; -const Args = struct { - profile_dir: ?[]const u8 = null, - paired: bool = false, - tuning: zml.paged_attention.cute.SparseTuning = .{}, - sparse: bool = false, - stress: bool = false, - all_backends: bool = false, - backend: ?fixtures.Backend = null, - case: ?[]const u8 = null, - dtype: zml.DataType = .bf16, - warmup: usize = 10, - samples: usize = 101, -}; - pub fn main(init: std.process.Init) !void { - const args = try parseArgs(init); const platform = try zml.Platform.auto(init.gpa, init.io, .{}); defer platform.deinit(init.gpa, init.io); + if (platform.target != .cuda) return error.CudaRequired; const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; - std.debug.print("PAGED_ATTENTION dtype={s} page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ - @tagName(args.dtype), cc.major, cc.minor, platform.fmtVerbose(), - }); - std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload, reference validation and output destruction)\n", .{platform.devices.len}); - std.debug.print("TUNING {any}\n", .{args.tuning}); - const suite: []const cases.Case = if (args.stress) &cases.sparseStressBenchmarks else if (args.sparse) &cases.sparseBenchmarks else &cases.benchmarks; + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, + cc.minor, + platform.fmtVerbose(), + }); + + std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); var failures: usize = 0; - var selected: usize = 0; - for (suite, 0..) |c, index| { - if (args.case) |filter| { - const wanted_index: ?usize = std.fmt.parseInt(usize, filter, 10) catch null; - if (wanted_index) |wanted| { - if (wanted != index) continue; - } else if (!std.mem.eql(u8, filter, c.name)) continue; - } - selected += 1; - if (args.paired) { - try runPaired(init.gpa, init.io, platform, mesh, c, index, args); - continue; - } - const backends = std.enums.values(fixtures.Backend); - for (0..backends.len) |rank| { - const backend = backends[(rank + index) % backends.len]; - if (args.backend) |wanted| { - if (wanted != backend) continue; - } else if (!args.sparse and !args.all_backends and backend != .cute) continue; - // Triton's GB300 dot accumulator has a documented 16K-element - // tile limit. Keep optional stress runs from faulting the device. - if (backend == .triton and c.mask == .indices and cc.major == 10 and cc.minor == 3 and c.group * c.head_dim > 16 * 1024) { - std.debug.print("SKIP index={d} name={s} backend=triton reason=gb300_triton_head_tile_limit\n", .{ index, c.name }); - continue; - } - std.debug.print("CASE index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} mask={s} backend={s}\n", .{ - index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(c.mask), @tagName(backend), - }); - runBenchmark(init.gpa, init.io, platform, mesh, c, index, backend, args) catch |err| { - failures += 1; - std.debug.print("FAIL error={s}\n", .{@errorName(err)}); - }; - } + for (cases.benchmarks) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + @tagName(c.mask), + }); + + runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; } - if (selected == 0) return error.UnknownCase; + if (failures != 0) return error.BenchmarkFailed; } -fn parseArgs(init: std.process.Init) !Args { - var result: Args = .{}; - var it = init.minimal.args.iterate(); - _ = it.next(); - while (it.next()) |arg| { - if (std.mem.eql(u8, arg, "--sparse")) { - result.sparse = true; - } else if (std.mem.eql(u8, arg, "--paired")) { - result.paired = true; - } else if (std.mem.eql(u8, arg, "--stress")) { - result.sparse = true; - result.stress = true; - } else if (std.mem.startsWith(u8, arg, "--tune-head-rows=")) { - result.tuning.headRows = try std.fmt.parseInt(i64, arg["--tune-head-rows=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-warps=")) { - result.tuning.warps = try std.fmt.parseInt(i64, arg["--tune-warps=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-tile=")) { - result.tuning.kvTile = try std.fmt.parseInt(i64, arg["--tune-tile=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-splits=")) { - result.tuning.splits = try std.fmt.parseInt(i64, arg["--tune-splits=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-merge-lanes=")) { - result.tuning.mergeLanes = try std.fmt.parseInt(i64, arg["--tune-merge-lanes=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-merge-vector=")) { - result.tuning.mergeVector = try std.fmt.parseInt(i64, arg["--tune-merge-vector=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--tune-merge-parts=")) { - result.tuning.mergeParts = try std.fmt.parseInt(i64, arg["--tune-merge-parts=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--backend=")) { - const value = arg["--backend=".len..]; - result.all_backends = std.mem.eql(u8, value, "all"); - result.backend = if (std.mem.eql(u8, value, "all")) null else std.meta.stringToEnum(fixtures.Backend, value) orelse return error.InvalidBackend; - } else if (std.mem.startsWith(u8, arg, "--case=")) { - result.case = arg["--case=".len..]; - } else if (std.mem.startsWith(u8, arg, "--dtype=")) { - result.dtype = std.meta.stringToEnum(zml.DataType, arg["--dtype=".len..]) orelse return error.InvalidDType; - if (result.dtype != .bf16 and result.dtype != .f16) return error.InvalidDType; - } else if (std.mem.startsWith(u8, arg, "--profile-dir=")) { - result.profile_dir = arg["--profile-dir=".len..]; - } else if (std.mem.startsWith(u8, arg, "--warmup=")) { - result.warmup = try std.fmt.parseInt(usize, arg["--warmup=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--samples=")) { - result.samples = try std.fmt.parseInt(usize, arg["--samples=".len..], 10); - if (result.samples == 0) return error.InvalidSamples; - } else if (std.mem.eql(u8, arg, "--help")) { - std.debug.print("paged attention benchmark [--sparse|--stress] [--paired] [--backend=cute|cute_generic|triton|all] [--case=INDEX|NAME] [--dtype=bf16|f16] [--warmup=10] [--samples=101] [--profile-dir=PATH] [--tune-head-rows=N --tune-warps=N --tune-tile=N --tune-splits=N --tune-merge-parts=N --tune-merge-vector=N --tune-merge-lanes=N ]\nSparse mode compares all three backends on identical seed-42 inputs; the default dense suite uses CuTe. Every measured case is first checked against the CPU reference. Use --stress --backend=NAME to isolate large-group backend resource limits.\n", .{}); - std.process.exit(0); - } else return error.UnknownArgument; - } - if (result.tuning.headRows) |v| if (v != 8 and v != 16) return error.InvalidTuning; - if (result.tuning.splits) |v| if (v < 1 or v > 128) return error.InvalidTuning; - if (result.tuning.mergeLanes) |v| if (std.mem.indexOfScalar(i64, &.{ 8, 16, 32 }, v) == null) return error.InvalidTuning; - if (result.tuning.mergeVector) |v| if (std.mem.indexOfScalar(i64, &.{ 0, 4, 8, 16 }, v) == null) return error.InvalidTuning; - if (result.tuning.warps) |v| if (std.mem.indexOfScalar(i64, &.{ 1, 2, 4, 8 }, v) == null) return error.InvalidTuning; - if (result.tuning.kvTile) |v| if (std.mem.indexOfScalar(i64, &.{ 16, 32, 64, 128 }, v) == null) return error.InvalidTuning; - if (result.tuning.mergeParts) |v| if (std.mem.indexOfScalar(i64, &.{ 4, 8, 16, 32 }, v) == null) return error.InvalidTuning; - if (result.paired and (result.stress or result.backend != null)) return error.InvalidPairedOptions; - return result; -} - -fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case, index: usize, backend: fixtures.Backend, args: Args) !void { - // Keep the number of KV heads per GPU constant. +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { + // Keep number of head per GPU constant. var global_case = c; global_case.kv_heads *= platform.devices.len; - var inputs = Inputs.initDType(global_case, args.dtype); + var inputs = Inputs.init(global_case); inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend, args.tuning }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); defer exe.deinit(); const compile_us = elapsedUs(compile_start, io); + var runner = try exe.runner(a); defer runner.deinit(a); + var arena = std.heap.ArenaAllocator.init(a); defer arena.deinit(); + const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); - if (args.dtype == .f16) fixture.quantizeF16(); - var buffers = try fixtures.upload(a, io, platform, inputs, fixture); + + var buffers = try fixtures.upload(io, platform, inputs, fixture); defer zml.Buffer.deinitAll(Inputs, &buffers); var output: zml.Buffer = undefined; - const max_error = validate: { - runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - defer output.deinit(); - var host = try output.toSliceAlloc(a, io); - defer host.free(a); - break :validate if (args.dtype == .f16) - try reference.check(a, global_case, fixture, host.items(f16)) - else - try reference.check(a, global_case, fixture, host.items(zml.floats.BFloat16)); - }; - for (0..args.warmup) |_| { + + // Warmup + for (0..5) |_| { runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); output.deinit(); } - const session = try std.fmt.allocPrint(a, "{s}-{d}-{s}", .{ @tagName(args.dtype), index, @tagName(backend) }); - defer a.free(session); - var profiler = if (args.profile_dir) |path| try platform.profiler(a, io, .{ - .repository_path = path, - .session_id = session, - .host_tracer_level = 0, - .device_tracer_level = 1, - .python_tracer_level = 0, - .raise_error_on_start_failure = true, - }) else null; - defer if (profiler) |*p| p.deinit(); - if (profiler) |*p| try p.start(); - const samples = try a.alloc(f64, args.samples); - defer a.free(samples); - for (samples) |*sample| { + + var samples: [101]f64 = undefined; + var sample_count: usize = 0; + var accumulated_us: f64 = 0; + while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { const call_start: std.Io.Timestamp = .now(io, .awake); runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - sample.* = elapsedUs(call_start, io); + samples[sample_count] = elapsedUs(call_start, io); + accumulated_us += samples[sample_count]; + sample_count += 1; output.deinit(); } - if (profiler) |*p| _ = try p.stop(); - std.mem.sort(f64, samples, {}, std.sort.asc(f64)); - const median = if (samples.len % 2 == 0) - (samples[samples.len / 2 - 1] + samples[samples.len / 2]) / 2 + + std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); + const median = if (sample_count % 2 == 0) + (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 else - samples[samples.len / 2]; - std.debug.print("RESULT index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_outputs max_abs={d:.6} compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ - index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(args.dtype), max_error, compile_us / 1000, samples.len, median, samples[samples.len / 10], samples[(samples.len - 1) * 9 / 10], + samples[sample_count / 2]; + + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + compile_us / 1000, + sample_count, + median, + samples[sample_count / 10], + samples[(sample_count - 1) * 9 / 10], }); } -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtures.Backend, tuning: zml.paged_attention.cute.SparseTuning) zml.Tensor { +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { const Call = struct { inputs: Inputs, case: cases.Case, - backend: fixtures.Backend, - tuning: zml.paged_attention.cute.SparseTuning, fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forwardBackendTuned(self.inputs, self.case, self.backend, self.tuning); + return fixtures.forward(self.inputs, self.case); } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c, .backend = backend, .tuning = tuning }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, @@ -233,88 +143,3 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh, backend: fixtur fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } - -// Keep all executables and inputs alive and rotate order for each sample round. -fn runPaired(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case, index: usize, args: Args) !void { - var global_case = c; - global_case.kv_heads *= platform.devices.len; - var inputs = Inputs.initDType(global_case, args.dtype); - inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); - inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); - inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); - inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); - var arena = std.heap.ArenaAllocator.init(a); - defer arena.deinit(); - const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); - if (args.dtype == .f16) fixture.quantizeF16(); - var buffers = try fixtures.upload(a, io, platform, inputs, fixture); - defer zml.Buffer.deinitAll(Inputs, &buffers); - const backends = std.enums.values(fixtures.Backend); - var executables: [3]zml.Exe = undefined; - var runners: [3]zml.Exe.Runner = undefined; - var compiled: usize = 0; - var initialized: usize = 0; - defer for (executables[0..compiled]) |e| e.deinit(); - defer for (runners[0..initialized]) |*r| r.deinit(a); - var errors: [3]f32 = undefined; - var compile_us: [3]f64 = undefined; - var samples: [3][]f64 = undefined; - for (&samples) |*v| v.* = try arena.allocator().alloc(f64, args.samples); - for (backends, 0..) |backend, i| { - const start: std.Io.Timestamp = .now(io, .awake); - const programName = try std.fmt.allocPrint(a, "paired_attention_{s}", .{@tagName(backend)}); - defer a.free(programName); - executables[i] = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh, backend, args.tuning }, .{ .program_name = programName, .meshes = &.{mesh} }); - compiled += 1; - compile_us[i] = elapsedUs(start, io); - runners[i] = try executables[i].runner(a); - initialized += 1; - var output: zml.Buffer = undefined; - runners[i].run(io, .{buffers}, .{&output}, .{ .wait = true }); - defer output.deinit(); - var host = try output.toSliceAlloc(a, io); - defer host.free(a); - errors[i] = if (args.dtype == .f16) try reference.check(a, global_case, fixture, host.items(f16)) else try reference.check(a, global_case, fixture, host.items(zml.floats.BFloat16)); - } - const session = try std.fmt.allocPrint(a, "{s}-{d}-paired", .{ @tagName(args.dtype), index }); - defer a.free(session); - var profiler = if (args.profile_dir) |path| try platform.profiler(a, io, .{ - .repository_path = path, - .session_id = session, - .host_tracer_level = 0, - .device_tracer_level = 1, - .python_tracer_level = 0, - .raise_error_on_start_failure = true, - }) else null; - defer if (profiler) |*p| p.deinit(); - for (0..args.warmup + args.samples) |round| { - if (round == args.warmup) if (profiler) |*p| try p.start(); - for (0..backends.len) |rank| { - const i = (rank + round) % backends.len; - var output: zml.Buffer = undefined; - const start: std.Io.Timestamp = .now(io, .awake); - runners[i].run(io, .{buffers}, .{&output}, .{ .wait = true }); - const us = elapsedUs(start, io); - if (round >= args.warmup) samples[i][round - args.warmup] = us; - output.deinit(); - } - } - if (profiler) |*p| _ = try p.stop(); - const deltas = try arena.allocator().alloc(f64, args.samples); - const ratios = try arena.allocator().alloc(f64, args.samples); - const cute_index = @backingInt(fixtures.Backend.cute); - const triton_index = @backingInt(fixtures.Backend.triton); - for (deltas, ratios, 0..) |*delta, *ratio, i| { - delta.* = samples[cute_index][i] - samples[triton_index][i]; - ratio.* = samples[triton_index][i] / samples[cute_index][i]; - } - std.mem.sort(f64, deltas, {}, std.sort.asc(f64)); - std.mem.sort(f64, ratios, {}, std.sort.asc(f64)); - std.debug.print("PAIR index={d} compute_dtype={s} n={d} cute_minus_triton_median_us={d:.3} triton_over_cute_median={d:.6} delta_p10_us={d:.3} delta_p90_us={d:.3}\n", .{ index, @tagName(args.dtype), deltas.len, deltas[deltas.len / 2], ratios[ratios.len / 2], deltas[deltas.len / 10], deltas[(deltas.len - 1) * 9 / 10] }); - for (backends, 0..) |backend, i| { - const values = samples[i]; - std.mem.sort(f64, values, {}, std.sort.asc(f64)); - const median = if (values.len % 2 == 0) (values[values.len / 2 - 1] + values[values.len / 2]) / 2 else values[values.len / 2]; - std.debug.print("RESULT index={d} name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_outputs paired=true max_abs={d:.6} compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ index, c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(args.dtype), errors[i], compile_us[i] / 1000, values.len, median, values[values.len / 10], values[(values.len - 1) * 9 / 10] }); - } -} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index 35c7c38f..e8dedeaa 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -15,19 +15,16 @@ pub const Case = struct { kv_len: usize = 65, kv_capacity: ?usize = null, inactive: usize = 0, - token_capacity: ?usize = null, shuffle_pages: bool = false, skew: bool = false, mask: Mask = .causal, - topk: usize = 8, empty_indices: bool = false, window: u32 = 17, sink: bool = false, - sink_value: ?f32 = null, scale: ?f32 = null, pub fn tokens(c: Case) usize { - return c.token_capacity orelse (c.prefill * c.chunk + c.batch - c.prefill - c.inactive); + return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; } pub fn pages(c: Case) usize { return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; @@ -104,74 +101,3 @@ pub const benchmarks = blk: { } break :blk result; }; - -// Selection-slot tails, GQA head tails, and logical-to-physical page gathers -// vary independently. Every nonempty fixture includes duplicate/invalid slots. -pub const sparseCorrectness = [_]Case{ - .{ .name = "sparse_decode_inactive", .head_dim = 64, .group = 5, .batch = 5, .inactive = 2, .token_capacity = 9, .mask = .indices, .topk = 65, .shuffle_pages = true }, - .{ .name = "sparse_tiny_decode_inactive", .head_dim = 128, .group = 1, .batch = 5, .inactive = 2, .token_capacity = 9, .mask = .indices, .topk = 8, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_tiny_d32_sink", .head_dim = 32, .group = 1, .mask = .indices, .topk = 1, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_tiny_d64_disabled_sink", .head_dim = 64, .group = 1, .mask = .indices, .topk = 7, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, - .{ .name = "sparse_tiny_d128_empty", .head_dim = 128, .group = 1, .mask = .indices, .topk = 8, .empty_indices = true }, - .{ .name = "sparse_tiny_d256_capacity", .head_dim = 256, .group = 1, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .mask = .indices, .topk = 5, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_tiny_d512_scale", .head_dim = 512, .group = 1, .mask = .indices, .topk = 8, .scale = -0.125, .shuffle_pages = true }, - .{ .name = "sparse_tiny_mha", .head_dim = 128, .group = 1, .mask = .indices, .topk = 8, .shuffle_pages = true }, - .{ .name = "sparse_d32_mha", .head_dim = 32, .group = 1, .mask = .indices, .topk = 32 }, - .{ .name = "sparse_d64_mqa", .head_dim = 64, .kv_heads = 1, .group = 8, .mask = .indices, .topk = 65 }, - .{ .name = "sparse_d128_mixed", .head_dim = 128, .prefill = 1, .chunk = 9, .mask = .indices, .topk = 65, .shuffle_pages = true }, - .{ .name = "sparse_d256_mixed_sink", .head_dim = 256, .prefill = 1, .chunk = 3, .mask = .indices, .topk = 32, .sink = true }, - .{ .name = "sparse_d512", .head_dim = 512, .mask = .indices, .topk = 65, .shuffle_pages = true }, - .{ .name = "sparse_group5", .head_dim = 64, .group = 5, .prefill = 1, .chunk = 5, .mask = .indices, .topk = 65 }, - .{ .name = "sparse_group19", .head_dim = 128, .group = 19, .mask = .indices, .topk = 65, .sink = true }, - .{ .name = "sparse_d512_group19", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 19, .mask = .indices, .topk = 32 }, - .{ .name = "sparse_long_selection", .head_dim = 128, .batch = 1, .kv_len = 2049, .mask = .indices, .topk = 512, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_long_tail", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .indices, .topk = 513, .shuffle_pages = true }, - .{ .name = "sparse_split_tail", .head_dim = 64, .batch = 1, .kv_len = 4097, .mask = .indices, .topk = 1025, .sink = true }, - .{ .name = "sparse_all_invalid", .head_dim = 128, .prefill = 1, .chunk = 3, .mask = .indices, .topk = 65, .empty_indices = true }, - .{ .name = "sparse_all_invalid_sink", .head_dim = 256, .mask = .indices, .topk = 512, .empty_indices = true, .sink = true }, - .{ .name = "sparse_skew_inactive", .head_dim = 128, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 5, .kv_len = 129, .mask = .indices, .topk = 65, .skew = true, .shuffle_pages = true }, - .{ .name = "sparse_spare_capacity", .head_dim = 128, .batch = 4, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 16, .kv_len = 129, .mask = .indices, .topk = 65, .shuffle_pages = true }, - .{ .name = "sparse_negative_scale", .head_dim = 64, .mask = .indices, .topk = 32, .scale = -0.125 }, - .{ .name = "sparse_all_invalid_disabled_sink", .head_dim = 128, .mask = .indices, .topk = 8, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, - .{ .name = "sparse_zero_scale", .head_dim = 128, .mask = .indices, .topk = 65, .scale = 0, .sink = true }, -}; - -pub const sparseBenchmarks = blk: { - var result: [25]Case = undefined; - var i: usize = 0; - for ([_]usize{ 64, 128, 256, 512 }) |dim| { - for ([_]usize{ 1, 8 }) |batch| { - for ([_]bool{ false, true }) |mixed| { - result[i] = .{ - .name = "sparse", - .head_dim = dim, - .batch = batch, - .prefill = if (mixed) @max(1, batch / 4) else 0, - .chunk = 16, - .kv_len = 8192, - .mask = .indices, - .topk = 512, - .shuffle_pages = true, - }; - i += 1; - } - } - } - result[16] = .{ .name = "sparse_short", .head_dim = 128, .batch = 8, .kv_len = 8192, .mask = .indices, .topk = 32, .shuffle_pages = true }; - result[17] = .{ .name = "sparse_odd_tail", .head_dim = 128, .batch = 8, .group = 8, .kv_len = 8192, .mask = .indices, .topk = 65, .shuffle_pages = true }; - result[18] = .{ .name = "sparse_large_gqa", .head_dim = 128, .batch = 1, .group = 19, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }; - result[19] = .{ .name = "sparse_mha", .head_dim = 128, .batch = 8, .group = 1, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; - result[20] = .{ .name = "sparse_batch64", .head_dim = 128, .batch = 64, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; - result[21] = .{ .name = "sparse_batch128", .head_dim = 128, .batch = 128, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; - result[22] = .{ .name = "sparse_tiny_mha", .head_dim = 128, .batch = 8, .group = 1, .kv_len = 8192, .mask = .indices, .topk = 8, .shuffle_pages = true }; - result[23] = .{ .name = "sparse_gqa32", .head_dim = 128, .batch = 8, .group = 32, .kv_len = 8192, .mask = .indices, .topk = 512, .shuffle_pages = true }; - result[24] = .{ .name = "sparse_mqa16_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 16, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }; - break :blk result; -}; - -// These geometries expose backend resource limits; keep them separately -// selectable so one unsupported backend cannot interrupt the shared matrix. -pub const sparseStressBenchmarks = [_]Case{ - .{ .name = "sparse_mqa64_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 64, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }, - .{ .name = "sparse_mqa128_d512", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 128, .kv_len = 8192, .mask = .indices, .topk = 2048, .shuffle_pages = true }, -}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index b7894fb8..436b9d3a 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -27,7 +27,7 @@ test "paged attention correctness" { c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), }); - runCorrectness(a, io, platform, c, .cute, .bf16) catch |err| { + runCorrectness(a, io, platform, c, false) catch |err| { failures += 1; std.debug.print("FAIL error={s}\n", .{@errorName(err)}); }; @@ -44,78 +44,19 @@ test "FP16 paged attention" { .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, - }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, .cute, .f16); + }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); } -test "indexed paged attention BF16 all backends" { - try runSparseCorrectness(.bf16); +fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { + var inputs = i; + inputs.q = i.q.convert(.f16); + inputs.k = i.k.convert(.f16); + inputs.v = i.v.convert(.f16); + return fixtures.forward(inputs, c).convert(.bf16); } -test "indexed paged attention FP16 all backends" { - try runSparseCorrectness(.f16); -} - -test "indexed long selections handle sinks tails and inactive capacity" { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.CudaRequired; - const suite = [_]cases.Case{ - .{ .name = "sparse_wide_sink_tail", .head_dim = 512, .batch = 1, .kv_heads = 1, .group = 16, .kv_len = 2049, .mask = .indices, .topk = 1025, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_wide_empty_capacity", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 5, .inactive = 1, .token_capacity = 6, .mask = .indices, .topk = 513, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, - .{ .name = "sparse_wide_prefill", .head_dim = 512, .group = 3, .prefill = 1, .chunk = 3, .kv_len = 2049, .mask = .indices, .topk = 513, .sink = true, .shuffle_pages = true }, - }; - for ([_]zml.DataType{ .bf16, .f16 }) |dtype| for (suite) |c| for (std.enums.values(fixtures.Backend)) |backend| { - std.debug.print("CASE name={s} backend={s} compute_dtype={s}\n", .{ c.name, @tagName(backend), @tagName(dtype) }); - try runCorrectness(std.testing.allocator, std.testing.io, platform, c, backend, dtype); - }; -} - -test "indexed eight-head tiles handle tails and packed capacity" { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.CudaRequired; - const suite = [_]cases.Case{ - .{ .name = "sparse_eight_head_mixed_tail", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 19, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .kv_len = 2049, .mask = .indices, .topk = 513, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_eight_head_small_tail", .head_dim = 64, .batch = 4, .group = 5, .prefill = 1, .inactive = 1, .chunk = 3, .token_capacity = 9, .mask = .indices, .topk = 65, .sink = true, .shuffle_pages = true }, - .{ .name = "sparse_eight_head_empty_capacity", .head_dim = 512, .batch = 3, .kv_heads = 1, .group = 16, .inactive = 1, .token_capacity = 6, .mask = .indices, .topk = 513, .empty_indices = true, .sink = true, .sink_value = -std.math.inf(f32) }, - }; - for ([_]zml.DataType{ .bf16, .f16 }) |dtype| for (suite) |c| { - std.debug.print("CASE name={s} backend=cute compute_dtype={s}\n", .{ c.name, @tagName(dtype) }); - const tuning: zml.paged_attention.cute.SparseTuning = .{ - .headRows = 8, - .warps = 8, - .kvTile = 32, - .splits = if (c.head_dim == 64) 3 else null, - .mergeLanes = if (c.head_dim == 64) 8 else 32, - .mergeVector = if (c.head_dim == 64) 4 else 8, - }; - try runCorrectnessTuned(std.testing.allocator, std.testing.io, platform, c, .cute, dtype, tuning); - }; -} - -fn runSparseCorrectness(dtype: zml.DataType) !void { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.CudaRequired; - var failures: usize = 0; - for (cases.sparseCorrectness) |c| { - for (std.enums.values(fixtures.Backend)) |backend| { - std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s}\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(dtype), - }); - runCorrectness(std.testing.allocator, std.testing.io, platform, c, backend, dtype) catch |err| { - failures += 1; - std.debug.print("FAIL error={s}\n", .{@errorName(err)}); - }; - } - } - try std.testing.expectEqual(@as(usize, 0), failures); -} - -fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, backend: fixtures.Backend, dtype: zml.DataType) !void { - return runCorrectnessTuned(a, io, platform, c, backend, dtype, .{}); -} - -fn runCorrectnessTuned(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, backend: fixtures.Backend, dtype: zml.DataType, tuning: zml.paged_attention.cute.SparseTuning) !void { - const inputs = Inputs.initDType(c, dtype); - const exe = try platform.compileFn(a, io, fixtures.forwardBackendTuned, .{ inputs, c, backend, tuning }, .{ .program_name = "paged_attention_harness" }); +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { + const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); defer exe.deinit(); var runner = try exe.runner(a); @@ -125,9 +66,17 @@ fn runCorrectnessTuned(a: std.mem.Allocator, io: std.Io, platform: *const zml.Pl defer arena.deinit(); const fixture = try fixtures.Fixture.init(arena.allocator(), c); - if (dtype == .f16) fixture.quantizeF16(); + if (compute_f16) { + // Exactly representable in both formats, so the existing CPU reference + // receives precisely the values used by the FP16 tensor-core path. + for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { + for (values) |*v| if (std.math.isFinite(v.toF32())) { + v.* = .fromF32(@round(v.toF32() * 64) / 64); + }; + } + } - var buffers = try fixtures.upload(a, io, platform, inputs, fixture); + var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); defer zml.Buffer.deinitAll(Inputs, &buffers); var output: zml.Buffer = undefined; @@ -137,18 +86,8 @@ fn runCorrectnessTuned(a: std.mem.Allocator, io: std.Io, platform: *const zml.Pl var host = try output.toSliceAlloc(a, io); defer host.free(a); - const max_error = if (dtype == .f16) - try reference.check(a, c, fixture, host.items(f16)) - else - try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); - if (backend != .triton) { - if (dtype == .f16) { - try reference.checkZeroTail(c, fixture, host.items(f16)); - } else { - try reference.checkZeroTail(c, fixture, host.items(zml.floats.BFloat16)); - } - } - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} topk={d} backend={s} compute_dtype={s} check=all_active_outputs inactive_tail={s} max_abs={d:.6}\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, c.topk, @tagName(backend), @tagName(dtype), if (backend == .triton) "unspecified" else "zero_checked", max_error, + const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, }); } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 733c9d68..ef78cc49 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -3,7 +3,6 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); pub const AttentionOptions = @import("../options.zig").AttentionOptions; const kernel = @import("attention.zig"); -const sparse = @import("sparse_attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); @@ -55,7 +54,6 @@ pub fn multiprocessorCount(platform: *const zml.Platform) usize { test { _ = kernel; - _ = sparse; _ = sm10x; _ = sm120; } @@ -66,38 +64,20 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -fn validateInputs(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor) void { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); -} - -fn sparseIndices(q: zml.Tensor, indices: zml.Tensor) zml.Tensor { - stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); - stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices require .b and .topk tags", .{}); - stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); - return indices.transpose(.{ .b, .topk }); -} - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { - validateInputs(q, k, v); - if (opts.mask == .indices) - return pagedSparseAttention(parameters, q, k, v, opts, .{}); - if (sm10x.isAvailable(zml.Compiler.current().platform)) + if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm10x.pagedAttention(parameters, q, k, v, opts); - if (sm120.isAvailable(zml.Compiler.current().platform)) + if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); - return pagedAttentionGeneric(parameters, q, k, v, opts); -} - -pub fn pagedAttentionGeneric(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { - validateInputs(q, k, v); const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); const indices = switch (opts.mask) { - .indices => |indices| sparseIndices(q_, indices), + .indices => |indices| indices.transpose(.{ .b, .topk }), else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), }; const cfg: kernel.Config = .{ @@ -133,126 +113,3 @@ pub fn pagedAttentionGeneric(parameters: Parameters, q: zml.Tensor, k: zml.Tenso .block = .{ 32, 1, 1 }, }).out.transpose(q.shape()); } - -// Overrides let the kernel benchmark evaluate launch geometry without changing -// the production policy or either comparison backend. -pub const SparseTuning = struct { - headRows: ?i64 = null, - warps: ?i64 = null, - kvTile: ?i64 = null, - splits: ?i64 = null, - mergeParts: ?i64 = null, - mergeVector: ?i64 = null, - mergeLanes: ?i64 = null, -}; - -pub fn pagedAttentionTuned(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions, tuning: SparseTuning) zml.Tensor { - validateInputs(q, k, v); - stdx.debug.assert(opts.mask == .indices, "Sparse tuning requires indexed attention", .{}); - return pagedSparseAttention(parameters, q, k, v, opts, tuning); -} - -fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions, tuning: SparseTuning) zml.Tensor { - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const indices = sparseIndices(q_, opts.mask.indices); - const all_decode = !parameters.options_.is_prefill; - const short_selection = all_decode and q_.dim(.hd) <= 128 and indices.dim(.topk) <= 128; - const fine_tile = all_decode and q_.dim(.b) == 1 and ((q_.dim(.hd) == 512 and indices.dim(.topk) <= 512) or (q_.dim(.hd) >= 128 and q_.dim(.hd) < 512 and q_.dim(.hg) <= 16)); - const split_features = all_decode and (q_.dim(.hd) == 512 or (fine_tile and indices.dim(.topk) > 128)); - var cfg: sparse.Config = .{ - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .topk = indices.dim(.topk), - .dtype = zml.kernel.cute.from(q_.dtype()), - .kv_tile = if (short_selection) (if (indices.dim(.topk) <= 32) @as(i64, 32) else 128) else if (fine_tile) 16 else if (all_decode or q_.dim(.hd) == 512) 32 else 64, - .warps = if (split_features) 8 else 4, - .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), - .sink = opts.sink != null, - .decode = all_decode and parameters.options_.max_seqlen_q == 1, - }; - const platform = zml.Compiler.current().platform; - const cc = zml.platform.cuda.computeCapability(platform).?; - const narrowDecode = cfg.decode and cfg.tokens == 1 and cfg.batch == 1 and - ((cfg.head_dim == 64 and cfg.group <= 8 and cfg.topk > 128) or - (cfg.head_dim == 512 and cfg.group <= 16 and cfg.topk > 512)); - // Underfilled GB300 grids benefit from eight real query rows per CTA. - // Vector merges amortize each partition's softmax state across features. - const narrowWork = cfg.kv_heads * @divTrunc(cfg.group + 7, 8) * @divTrunc(cfg.topk + 31, 32); - if (cc.major == 10 and cc.minor == 3 and narrowDecode and narrowWork <= @as(i64, @intCast(multiprocessorCount(platform)))) { - cfg.head_rows = 8; - cfg.warps = 8; - cfg.kv_tile = 32; - cfg.merge_vector = if (cfg.head_dim == 512) 8 else 4; - cfg.merge_lanes = if (cfg.head_dim == 512) 32 else 8; - } - if (tuning.headRows) |rows| { - stdx.debug.assert(rows == 8 or rows == 16, "Invalid sparse head tile", .{}); - cfg.head_rows = rows; - } - if (tuning.warps) |warps| cfg.warps = warps; - if (tuning.kvTile) |tile| cfg.kv_tile = tile; - cfg.merge_parts = if (cfg.head_dim >= 128 and cfg.head_dim <= 256 and cfg.tokens * cfg.kv_heads * cfg.group <= 16) 16 else 8; - if (tuning.mergeParts) |parts| cfg.merge_parts = parts; - if (tuning.mergeLanes) |lanes| cfg.merge_lanes = lanes; - stdx.debug.assert(cfg.merge_lanes == 8 or cfg.merge_lanes == 16 or cfg.merge_lanes == 32, "Invalid sparse merge lanes", .{}); - if (tuning.mergeVector) |features| { - stdx.debug.assert(features == 0 or ((features == 4 or features == 8 or features == 16) and cfg.head_dim >= @divExact(128, cfg.merge_lanes) * features), "Invalid sparse merge vector", .{}); - cfg.merge_vector = features; - } - stdx.debug.assert(@mod(cfg.head_dim, cfg.mergeFeatures()) == 0, "Sparse merge tile must divide the head dimension", .{}); - const work = @max(1, cfg.tokens * cfg.kv_heads * cfg.headTiles()); - // Short selections avoid a merge; longer decode selections expose one - // selected tile per partition until the device has enough parallel work. - const target = @as(i64, @intCast(multiprocessorCount(zml.Compiler.current().platform))) * 16; - const tiles = @divTrunc(cfg.topk + cfg.kv_tile - 1, cfg.kv_tile); - var desired: i64 = 1; - while (desired * work < target) desired *= 2; - cfg.splits = if (short_selection) 1 else if (all_decode) @max(1, @min(128, @min(tiles, desired))) else if (cfg.head_dim <= 128) 1 else @max(1, @min(32, @min(@divTrunc(@divTrunc(target, 8) + work - 1, work), @divTrunc(cfg.topk, 128)))); - if (tuning.splits) |splits| { - stdx.debug.assert(splits > 0 and splits <= tiles and splits <= 128, "Invalid sparse split count", .{}); - cfg.splits = splits; - } - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.scalar(0, .f32); - if (cfg.group == 1 and cfg.topk <= 8) return sparse.Tiny.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads), 1 }, - .block = .{ if (cfg.topk > 4) @as(i32, 256) else 128, 1, 1 }, - }).out.transpose(q.shape()); - const out = sparse.Kernel.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = if (cfg.splits > 1) zml.Shape.init(.{ cfg.tokens, cfg.kv_heads * cfg.group, cfg.splits, cfg.partialWidth() }, .f32) else q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.headTiles()), @intCast(cfg.splits) }, - .block = .{ @intCast(cfg.warps * 32), 1, 1 }, - }).out; - const result = if (cfg.splits > 1) sparse.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), @intCast(@divExact(cfg.head_dim, cfg.mergeFeatures())) }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(q.shape()); -} diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 15b6eb09..c0815d1d 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -4,8 +4,6 @@ const zml = @import("zml"); const cases = @import("cases.zig"); -pub const Backend = enum { cute, cute_generic, triton }; - pub const Fixture = struct { q: []zml.floats.BFloat16, k: []zml.floats.BFloat16, @@ -26,14 +24,14 @@ pub const Fixture = struct { .lengths = try a.alloc(i32, c.batch), .starts = try a.alloc(i32, c.batch + 1), .sinks = try a.alloc(f32, c.heads()), - .indices = try a.alloc(i32, c.tokens() * c.topk), + .indices = try a.alloc(i32, c.tokens() * cases.topk), }; var rng = std.Random.DefaultPrng.init(42); const random = rng.random(); for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); - for (f.sinks, 0..) |*v, h| v.* = c.sink_value orelse (@as(f32, @floatFromInt(h % 5)) - 2); + for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; @memset(f.indices, -1); f.starts[0] = 0; for (0..c.batch) |s| { @@ -57,15 +55,7 @@ pub const Fixture = struct { } for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { const l: i32 = @intCast(len); - const boundary = [_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }; - for (f.indices[t * c.topk ..][0..c.topk], 0..) |*index, j| { - index.* = if (j < boundary.len) boundary[j] else switch (j % 17) { - 0 => -1, - 1 => l, - 2 => 0, - else => @intCast((j * 37 + t * 13) % len), - }; - } + @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); } } if (c.empty_indices) { @@ -73,20 +63,9 @@ pub const Fixture = struct { @memset(f.k, .fromF32(std.math.nan(f32))); @memset(f.v, .fromF32(std.math.nan(f32))); } - const active_tokens: usize = @intCast(f.starts[c.batch]); - @memset(f.q[active_tokens * c.heads() * c.head_dim ..], .fromF32(std.math.nan(f32))); return f; } - pub fn quantizeF16(f: Fixture) void { - // Both host reference and native FP16 uploads receive identical values. - for ([_][]zml.floats.BFloat16{ f.q, f.k, f.v }) |values| { - for (values) |*v| if (std.math.isFinite(v.toF32())) { - v.* = .fromF32(@round(v.toF32() * 64) / 64); - }; - } - } - pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; @@ -104,19 +83,15 @@ pub const Inputs = struct { indices: zml.Tensor, pub fn init(c: cases.Case) Inputs { - return initDType(c, .bf16); - } - - pub fn initDType(c: cases.Case, dtype: zml.DataType) Inputs { return .{ - .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, dtype), - .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, dtype), - .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, dtype), + .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), + .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), .lengths = .init(.{ .b = c.batch }, .i32), .starts = .init(.{ .b = c.batch + 1 }, .i32), .sinks = .init(.{c.heads()}, .f32), - .indices = .init(.{ .b = c.tokens(), .topk = c.topk }, .i32), + .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), }; } @@ -136,61 +111,30 @@ pub const Inputs = struct { }; pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { - return forwardBackend(i, c, .cute); -} - -pub fn forwardBackend(i: Inputs, c: cases.Case, backend: Backend) zml.Tensor { - return forwardBackendTuned(i, c, backend, .{}); -} - -pub fn forwardBackendTuned(i: Inputs, c: cases.Case, backend: Backend, tuning: zml.paged_attention.cute.SparseTuning) zml.Tensor { - const parameters: zml.paged_attention.cute.Parameters = .{ - .block_table = i.table, - .seq_lens = i.lengths, - .query_start_len = i.starts, - .options_ = .{ - .batch_size = c.batch, - .max_num_pages = c.pages(), - .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, - .is_prefill = c.prefill > 0, - }, - }; - return switch (backend) { - .cute => if (c.mask == .indices) zml.paged_attention.cute.pagedAttentionTuned(parameters, i.q, i.k, i.v, i.options(c), tuning) else zml.paged_attention.cute.pagedAttention(parameters, i.q, i.k, i.v, i.options(c)), - .cute_generic => zml.paged_attention.cute.pagedAttentionGeneric(parameters, i.q, i.k, i.v, i.options(c)), - .triton => zml.paged_attention.triton.paged.pagedAttention(.{ + const parameters: zml.paged_attention.Parameters = .{ + .cute = .{ .block_table = i.table, .seq_lens = i.lengths, .query_start_len = i.starts, .options_ = .{ - .batch_size = @intCast(c.batch), - .max_num_pages = @intCast(c.pages()), - .max_seqlen_q = @intCast(if (c.prefill > 0) c.chunk else 1), + .batch_size = c.batch, + .max_num_pages = c.pages(), + .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, .is_prefill = c.prefill > 0, }, - }, i.q, i.k, i.v, i.options(c)), + }, }; + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); } -pub fn upload(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { +pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { var result: zml.Bufferized(Inputs) = undefined; var initialized: usize = 0; errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { if (index < initialized) @field(result, name).deinit(); }; inline for (@typeInfo(Inputs).@"struct".field_names) |name| { - if (comptime std.mem.eql(u8, name, "q") or std.mem.eql(u8, name, "k") or std.mem.eql(u8, name, "v")) { - if (@field(inputs, name).dtype() == .f16) { - const values = try a.alloc(f16, @field(f, name).len); - defer a.free(values); - for (values, @field(f, name)) |*dst, src| dst.* = @floatCast(src.toF32()); - @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(values)); - } else { - @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); - } - } else { - @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); - } + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); initialized += 1; } return result; diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig index b80fabee..1a030646 100644 --- a/zml/kernels/paged_attention/cute/reference.zig +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -10,7 +10,7 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u const start: usize = @intCast(f.starts[seq]); const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); const position = len - qlen + token - start; - const count = if (c.mask == .indices) c.topk else len; + const count = if (c.mask == .indices) cases.topk else len; const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); @@ -44,19 +44,15 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u } fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { - return if (c.mask == .indices) f.indices[token * c.topk + j] else @intCast(j); + return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); } -fn toF32(value: anytype) f32 { - return if (@TypeOf(value) == zml.floats.BFloat16) value.toF32() else @floatCast(value); -} - -pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: anytype) !f32 { +pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { const active_tokens: usize = @intCast(f.starts[c.batch]); for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { - if (!std.math.isFinite(toF32(v))) return error.NonFiniteOutput; + if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; } - const scores = try a.alloc(f32, @max(c.kv_len, c.topk)); + const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); defer a.free(scores); const expected = try a.alloc(f32, c.head_dim); defer a.free(expected); @@ -68,7 +64,7 @@ pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: a row(c, f, s, t, h, scores, expected); const base = (t * c.heads() + h) * c.head_dim; for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { - const got = toF32(got_bf16); + const got = got_bf16.toF32(); const err = @abs(want - got); max_error = @max(max_error, err); // Fixed BF16 error budget; never adapted to observed failures. @@ -84,13 +80,6 @@ pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: a return max_error; } -pub fn checkZeroTail(c: cases.Case, f: fixtures.Fixture, actual: anytype) !void { - const active_tokens: usize = @intCast(f.starts[c.batch]); - for (actual[active_tokens * c.heads() * c.head_dim ..]) |value| { - if (toF32(value) != 0) return error.NonzeroInactiveOutput; - } -} - test "reference single key, uniform causal scores, and sink" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -136,50 +125,3 @@ test "reference windows, noncausal, duplicate and invalid indices" { row(c, f, 0, 1, 0, &scores, &out); for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); } - -test "reference indexed selection tail, FP16 output and unused query capacity" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const a = arena.allocator(); - const c: cases.Case = .{ - .head_dim = 64, - .batch = 1, - .kv_heads = 1, - .group = 1, - .prefill = 1, - .chunk = 2, - .kv_len = 2, - .mask = .indices, - .topk = 65, - .token_capacity = 4, - }; - const f = try fixtures.Fixture.init(a, c); - @memset(f.q[0 .. 2 * 64], .fromF32(0)); - @memset(f.v[0..64], .fromF32(2)); - @memset(f.v[64..128], .fromF32(4)); - @memset(f.indices, -1); - f.indices[64] = 1; - f.indices[c.topk + 64] = 0; - const actual = try a.alloc(f16, c.tokens() * c.heads() * c.head_dim); - @memset(actual[0..64], 4); - @memset(actual[64..128], 2); - @memset(actual[128..], 0); - try std.testing.expectEqual(@as(f32, 0), try check(a, c, f, actual)); - try checkZeroTail(c, f, actual); - actual[128] = std.math.nan(f16); - try std.testing.expectError(error.NonzeroInactiveOutput, checkZeroTail(c, f, actual)); -} - -test "reference all invalid indexed slots with sink ignore poisoned cache" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .mask = .indices, .topk = 65, .empty_indices = true, .sink = true }; - const f = try fixtures.Fixture.init(arena.allocator(), c); - var scores: [65]f32 = undefined; - var out: [64]f32 = undefined; - row(c, f, 0, 0, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); - @memset(f.sinks, -std.math.inf(f32)); - row(c, f, 0, 0, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); -} diff --git a/zml/kernels/paged_attention/cute/sparse_attention.zig b/zml/kernels/paged_attention/cute/sparse_attention.zig deleted file mode 100644 index c19d7df6..00000000 --- a/zml/kernels/paged_attention/cute/sparse_attention.zig +++ /dev/null @@ -1,1005 +0,0 @@ -// Decode CTA width bounds the output fragment at 64 FP32 values per thread. -// Rows are query heads of one token sharing a KV head. -const std = @import("std"); -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); -const vector = @import("mlir/dialects/vector"); - -const zml = @import("../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub const Config = struct { - head_dim: i64, - tokens: i64, - batch: i64, - kv_heads: i64, - group: i64, - pages_per_seq: i64, - page_size: i64, - topk: i64, - dtype: cute.DType, - kv_tile: i64, - warps: i64 = 4, - scale: f32, - sink: bool, - decode: bool = false, - head_rows: i64 = 16, - merge_parts: i64 = 8, - merge_vector: i64 = 0, - merge_lanes: i64 = 32, - splits: i64 = 1, - - pub fn rows(c: Config) i64 { - return c.head_rows; - } - pub fn headTiles(c: Config) i64 { - return @divTrunc(c.group + c.head_rows - 1, c.head_rows); - } - fn kvBase(c: Config) i64 { - return c.rows() * c.head_dim * 2; - } - pub fn partialWidth(c: Config) i64 { - // Align each split to a 32-byte sector, including max/denominator. - return c.head_dim + 8; - } - pub fn mergeFeatures(c: Config) i64 { - return if (c.merge_vector > 0) @divExact(128, c.merge_lanes) * c.merge_vector else @divExact(128, c.merge_parts); - } - fn pBase(c: Config) i64 { - return c.kvBase() + c.kv_tile * c.head_dim * 2; - } - fn redBase(c: Config) i64 { - const key_warps = @min(c.warps, @min(@divTrunc(c.kv_tile, 8), 4)); - const feature_parts = @divExact(c.warps, key_warps); - // Wide-head QK partials share this allocation with the later FP16 P. - return c.pBase() + c.rows() * c.kv_tile * (if (feature_parts > 1) feature_parts * 4 else 2); - } - fn indexBase(c: Config) i64 { - return c.redBase() + 2 * 64 * 4; - } - fn sharedBytes(c: Config) i64 { - return c.indexBase() + c.kv_tile * 8; - } -}; - -pub const Kernel = cute.Kernel(Config, .{ - .name = "paged_attention_cute_indexed_grouped", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, - .outputs = &.{"out"}, - .run = run, -}); - -// Small MHA selections use SIMT: warps evaluate independent selected -// slots, then reduce their weighted values within one CTA and one launch. -pub const Tiny = cute.Kernel(Config, .{ - .name = "paged_attention_cute_indexed_tiny", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, - .outputs = &.{"out"}, - .run = runTiny, -}); - -// In a decode-only batch each sequence owns zero or one query. Equal total -// active tokens and sequence count proves token == sequence; spare capacity -// and inactive sequences retain the general upper-bound lookup. -fn findSequence(b: *B, c: Config, starts: V, token: V, active_tokens: V) V { - var fast = b.openIfElse(if (c.decode) active_tokens.eq(c.batch) else b.constant(.i1, 0), .{.i32}); - fast.yieldThen(.{token}); - var sequence = b.openWhile(.{ b.constant(.i32, 0), b.constant(.i32, c.batch) }, .{ .i32, .i32 }); - sequence.yieldBefore(sequence.before_carried[0].lt(sequence.before_carried[1]), sequence.before_carried); - const left = sequence.after_carried[0]; - const right = sequence.after_carried[1]; - const mid = left.add(right).div(2); - const before = loadI32(b, starts, mid).le(token); - sequence.yieldAfter(.{ b.select(before, mid.add(1), left), b.select(before, right, mid) }); - fast.yieldElse(.{sequence.results[0].sub(1)}); - return fast.results[0]; -} - -fn loadHalf(b: *B, address: V, comptime dt: cute.DType) V { - return b.emit(llvm.load(b.ctx, b.addressPtr(address, .global).inner, dt.toMlir(b.ctx), .{ .alignment = 2 }, b.loc())).to(.f32); -} - -fn runTiny(b: *B, c: Config) cute.FinishError!void { - std.debug.assert(c.group == 1 and c.topk <= 8); - switch (c.head_dim) { - inline 32, 64, 128, 256, 512 => |d| switch (c.dtype) { - inline .bf16, .f16 => |dt| try runTinyDimension(b, c, d, dt), - else => unreachable, - }, - else => unreachable, - } -} - -fn runTinyDimension(b: *B, c: Config, comptime d: usize, comptime dt: cute.DType) cute.FinishError!void { - const chunks = d / 32; - const warps: usize = if (c.topk > 4) 8 else 4; - const slots: usize = 8 / warps; - const a = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .table = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .indices = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = dt }, - }); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - const tid = b.threadIdx().x; - const lane = tid.rem(32); - const warp = tid.div(32); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const starts = b.ptrToInt(a.starts, .i64); - const active_tokens = loadI32(b, starts, b.constant(.i32, c.batch)); - const out = b.ptrToInt(a.out, .i64).add(token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2)); - var inactive = b.openIf(token.ge(active_tokens)); - { - var dims = b.openFor(tid, d, warps * 32, .{}); - b.storeHalf(out.add(dims.iv.to(.i64).mul(2)), .global, b.constant(.i32, 0)); - dims.close(); - inactive.close(); - } - var active = b.openIf(token.lt(active_tokens)); - { - const seq = findSequence(b, c, starts, token, active_tokens); - const len = loadI32(b, b.ptrToInt(a.lengths, .i64), seq); - const q = b.ptrToInt(a.q, .i64).add(token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2)); - const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(32 + warps * d * 4), 16), .i32); - var query: [chunks]V = undefined; - for (&query, 0..) |*value, i| value.* = loadHalf(b, q.add(lane.add(i * 32).to(.i64).mul(2)), dt); - var rows: [2]V = undefined; - var valid: [2]V = undefined; - for (0..slots) |i| { - const slot = warp.add(i * warps); - var selected = b.openIfElse(slot.lt(c.topk), .{.i32}); - selected.yieldThen(.{loadI32(b, b.ptrToInt(a.indices, .i64), token.to(.i64).mul(c.topk).add(slot.to(.i64)))}); - selected.yieldElse(.{b.constant(.i32, -1)}); - const key = selected.results[0]; - valid[i] = key.ge(0).bitAnd(key.lt(len)); - var entry = b.openIfElse(valid[i], .{ .f32, .i64 }); - { - const page = loadI32(b, b.ptrToInt(a.table, .i64), seq.mul(c.pages_per_seq).add(key.div(c.page_size))); - const row = page.to(.i64).mul(c.page_size).add(key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(head.to(.i64)).mul(d * 2); - var score = zero; - for (query, 0..) |value, j| score = score.add(value.mul(loadHalf(b, b.ptrToInt(a.k, .i64).add(row).add(lane.add(j * 32).to(.i64).mul(2)), dt))); - inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); - entry.yieldThen(.{ score.mul(c.scale * std.math.log2e), row }); - } - entry.yieldElse(.{ neg_inf, b.constant(.i64, 0) }); - rows[i] = entry.results[1]; - var leader = b.openIf(lane.eq(0)); - b.storeWords(1, shared.add(slot.mul(4)), .shared, .{entry.results[0].bitCast(.i32)}); - leader.close(); - } - b.syncThreads(); - const sink = if (c.sink) b.loadWords(1, b.ptrToInt(a.sink, .i64).add(head.to(.i64).mul(4)), .global, .{})[0].bitCast(.f32).mul(std.math.log2e) else neg_inf; - var maximum = sink; - var scores: [8]V = undefined; - for (&scores, 0..) |*score, i| { - score.* = b.loadWords(1, shared.add(i * 4), .shared, .{})[0].bitCast(.f32); - maximum = maximum.maximum(score.*); - } - maximum = maximum.maximum(b.constant(.f32, -std.math.floatMax(f32))); - var denominator = if (c.sink) b.exp2Approx(sink.sub(maximum)) else zero; - for (scores) |score| denominator = denominator.add(b.exp2Approx(score.sub(maximum))); - const norm = b.select(denominator.gt(0), b.rcpApprox(denominator), zero); - var output: [chunks]V = @splat(zero); - for (0..slots) |i| { - const score = b.loadWords(1, shared.add(warp.add(i * warps).mul(4)), .shared, .{})[0].bitCast(.f32); - const weight = b.exp2Approx(score.sub(maximum)).mul(norm); - var entry = b.openIfElse(valid[i], @as([chunks]V, @splat(zero))); - var values: [chunks]V = undefined; - for (&values, 0..) |*value, j| value.* = output[j].add(weight.mul(loadHalf(b, b.ptrToInt(a.v, .i64).add(rows[i]).add(lane.add(j * 32).to(.i64).mul(2)), dt))); - entry.yieldThen(values); - entry.yieldElse(output); - output = entry.results; - } - for (output, 0..) |value, i| b.storeWords(1, shared.add(32).add(warp.mul(d).add(lane.add(i * 32)).mul(4)), .shared, .{value.bitCast(.i32)}); - b.syncThreads(); - var dims = b.openFor(tid, d, warps * 32, .{}); - var value = zero; - for (0..warps) |i| value = value.add(b.loadWords(1, shared.add(32 + i * d * 4).add(dims.iv.mul(4)), .shared, .{})[0].bitCast(.f32)); - b.storeHalf(out.add(dims.iv.to(.i64).mul(2)), .global, value.to(dt).bitCast(.i16).to(.i32)); - dims.close(); - active.close(); - } -} - -fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { - var result: [n * 4]V = undefined; - for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); - return result; -} - -fn packAcc(b: *B, values: [4]V) V { - return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); -} - -fn loadI32(b: *B, ptr: V, index: V) V { - return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; -} - -// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. -fn swizzle(row: V, col: V, width: i64) V { - return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); -} - -fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { - var result: [n]V = undefined; - for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); - return result; -} - -fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { - const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ - (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); - return unpackStruct(b, n, result, .i32); -} - -fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { - const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ - "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); - return unpackStruct(b, 4, result, .f32); -} - -fn run(b: *B, c: Config) cute.FinishError!void { - switch (c.head_dim) { - inline 32, 64, 128, 256 => |d| switch (c.warps) { - inline 1, 2, 4 => |w| try runDimension(b, c, d, w), - 8 => if (d >= 64) try runDimension(b, c, d, 8) else unreachable, - else => unreachable, - }, - 512 => { - const d = 512; - switch (c.warps) { - 4 => try runDimension(b, c, d, 4), - 8 => try runDimension(b, c, d, 8), - else => unreachable, - } - }, - else => unreachable, - } -} - -fn runDimension(b: *B, c: Config, comptime d: usize, comptime w: usize) cute.FinishError!void { - switch (c.kv_tile) { - 16 => switch (c.dtype) { - inline .bf16, .f16 => |dt| try runTile(b, c, d, w, 16, dt), - else => unreachable, - }, - inline 32, 64, 128 => |n| switch (c.dtype) { - inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), - else => unreachable, - }, - else => unreachable, - } -} - -const TileContext = struct { - b: *B, - c: Config, - q: V, - k: V, - v: V, - table: V, - indices: V, - sink: V, - out: V, - shared: V, - tid: V, - lane: V, - warp: V, - seq: V, - head: V, - first_row: V, - token: V, - klen: V, - split: V, - - fn sync(t: TileContext) void { - if (t.c.warps == 1) { - t.b.syncWarp(); - } else t.b.syncThreads(); - } - - // Resolve each selected slot once, then share its byte offset across - // every head, feature vector, and both K/V loads. Invalid logical indices - // never dereference the page table; cp.async zero-fills their K/V rows. - fn gather(t: TileContext, key_base: V) void { - const b = t.b; - const c = t.c; - var selected = b.openFor(t.tid, c.kv_tile, c.warps * 32, .{}); - { - const address = t.shared.add(c.indexBase()).add(selected.iv.mul(8)); - b.emitVoid(llvm.store(b.ctx, b.constant(.i64, -1).inner, b.addressPtr(address, .shared).inner, .{ .alignment = 8 }, b.loc())); - const slot = key_base.add(selected.iv); - var in_range = b.openIf(slot.lt(c.topk)); - { - const key = loadI32(b, t.indices, t.token.to(.i64).mul(c.topk).add(slot.to(.i64))); - var valid = b.openIf(key.ge(0).bitAnd(key.lt(t.klen))); - { - const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(key.div(c.page_size))); - const resolvedOffset = page.to(.i64).mul(c.page_size).add(key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim * 2); - b.emitVoid(llvm.store(b.ctx, resolvedOffset.inner, b.addressPtr(address, .shared).inner, .{ .alignment = 8 }, b.loc())); - valid.close(); - } - in_range.close(); - } - selected.close(); - } - t.sync(); - } - - // Preserve offsets beyond 2 GiB in large paged caches. - fn byteOffset(t: TileContext, row: V) V { - const b = t.b; - const address = t.shared.add(t.c.indexBase()).add(row.mul(8)); - return b.emit(llvm.load(b.ctx, b.addressPtr(address, .shared).inner, cute.DType.i64.toMlir(b.ctx), .{ .alignment = 8 }, b.loc())); - } - - fn validSlot(t: TileContext, row: V) V { - return t.byteOffset(row).ge(0); - } - - fn loadKV(t: TileContext, source: V) void { - const vectors = @divExact(t.c.kv_tile * t.c.head_dim, 8); - const threads = t.c.warps * 32; - for (0..@intCast(@divTrunc(vectors + threads - 1, threads))) |i| { - const iv = t.tid.add(i * @as(usize, @intCast(threads))); - if (@mod(vectors, threads) == 0) { - t.copyKV(source, iv); - } else { - var valid = t.b.openIf(iv.lt(vectors)); - t.copyKV(source, iv); - valid.close(); - } - } - } - - fn copyKV(t: TileContext, source: V, iv: V) void { - const b = t.b; - const c = t.c; - const row = iv.div(@divExact(c.head_dim, 8)); - const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); - const offset = t.byteOffset(row); - const valid = offset.ge(0); - const safeOffset = b.select(valid, offset, b.constant(.i64, 0)); - b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(safeOffset).add(col.to(.i64).mul(2)), valid); - } - - fn loadQuery(t: TileContext) void { - const vectors = @divExact(t.c.rows() * t.c.head_dim, 8); - const threads = t.c.warps * 32; - for (0..@intCast(@divTrunc(vectors + threads - 1, threads))) |i| { - const iv = t.tid.add(i * @as(usize, @intCast(threads))); - if (@mod(vectors, threads) == 0) { - t.copyQuery(iv); - } else { - var valid = t.b.openIf(iv.lt(vectors)); - t.copyQuery(iv); - valid.close(); - } - } - } - - fn copyQuery(t: TileContext, iv: V) void { - const c = t.c; - const row = iv.div(@divExact(c.head_dim, 8)); - const col = iv.rem(@divExact(c.head_dim, 8)).mul(8); - const packed_row = t.first_row.add(row); - const valid = packed_row.lt(c.group).bitAnd(row.lt(c.head_rows)); - const safe_row = t.b.select(valid, packed_row, t.b.constant(.i32, 0)); - const offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(safe_row.to(.i64)).mul(c.head_dim).add(col.to(.i64)); - t.b.cpAsync16(t.shared.add(swizzle(row, col, c.head_dim)), t.q.add(offset.mul(2)), valid); - } - - fn loadRows(t: TileContext, base: V, rowBase: V, colBase: V, width: i64) [4]V { - if (t.c.head_rows == 8) { - const pair = matrixLoad(t.b, 2, false, base.add(swizzle(rowBase.add(t.lane.rem(8)), colBase.add(t.lane.div(8).rem(2).mul(8)), width))); - const zero = t.b.constant(.i32, 0); - return .{ pair[0], zero, pair[1], zero }; - } - return matrixLoad(t.b, 4, false, base.add(swizzle(rowBase.add(t.lane.rem(16)), colBase.add(t.lane.div(16).mul(8)), width))); - } - - fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { - const b = t.b; - var v = value; - inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); - const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); - var leader = b.openIf(t.lane.rem(4).eq(0)); - { - b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); - leader.close(); - } - // Callers synchronize once both row halves have been published. - } - - fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { - const b = t.b; - const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(row.mul(4)); - var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); - for (1..w) |i| { - const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); - result = if (maximum) result.maximum(v) else result.add(v); - } - return result; - } - - fn storePair(t: TileContext, row: V, dim: V, first: V, second: V, comptime dt: cute.DType) void { - const c = t.c; - var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); - if (c.splits > 1) { - offset = offset.mul(c.splits).add(t.split.to(.i64)).mul(c.partialWidth()).add(dim.to(.i64)); - t.b.storeWords(2, t.out.add(offset.mul(4)), .global, .{ first.bitCast(.i32), second.bitCast(.i32) }); - } else { - offset = offset.mul(c.head_dim).add(dim.to(.i64)); - t.b.storeWords(1, t.out.add(offset.mul(2)), .global, .{t.b.packHalf2(first, second, dt)}); - } - } - - fn store(t: TileContext, row: V, dim: V, value: V) void { - const c = t.c; - var offset = t.token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.to(.i64)); - if (c.splits > 1) { - offset = offset.mul(c.splits).add(t.split.to(.i64)).mul(c.partialWidth()).add(dim.to(.i64)); - t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); - } else { - offset = offset.mul(c.head_dim).add(dim.to(.i64)); - t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); - } - } -}; - -fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { - std.debug.assert(c.sharedBytes() <= 99 * 1024); - const a = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .table = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .indices = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, - }); - if (c.sharedBytes() > 48 * 1024) b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", .int(b.ctx, .i32, c.sharedBytes()))}))); - const token = b.blockIdx().x; - const first_row = b.blockIdx().y.rem(c.headTiles()).mul(c.head_rows); - const head = b.blockIdx().y.div(c.headTiles()); - const starts = b.ptrToInt(a.starts, .i64); - const active_tokens = loadI32(b, starts, b.constant(.i32, c.batch)); - // Unused capacity is observable in the public tensor and must be zero. - var inactive = b.openIf(token.ge(active_tokens)); - { - const out = b.ptrToInt(a.out, .i64); - var dims = b.openFor(b.threadIdx().x, c.rows() * (if (c.splits > 1) c.partialWidth() else c.head_dim), c.warps * 32, .{}); - { - const width = if (c.splits > 1) c.partialWidth() else c.head_dim; - const row = first_row.add(dims.iv.div(width)); - const dim = dims.iv.rem(width); - var valid = b.openIf(row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows)))); - { - var offset = token.to(.i64).mul(c.kv_heads).add(head.to(.i64)).mul(c.group).add(row.to(.i64)); - if (c.splits > 1) { - offset = offset.mul(c.splits).add(b.blockIdx().z.to(.i64)).mul(width).add(dim.to(.i64)); - const value = b.select(dim.eq(c.head_dim), b.constant(.f32, -std.math.inf(f32)), b.constant(.f32, 0)); - b.storeWords(1, out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); - } else { - b.storeHalf(out.add(offset.mul(width).add(dim.to(.i64)).mul(2)), .global, b.constant(.i32, 0)); - } - valid.close(); - } - dims.close(); - } - inactive.close(); - } - var active = b.openIf(token.lt(active_tokens)); - { - const tid = b.threadIdx().x; - const seq = findSequence(b, c, starts, token, active_tokens); - const t: TileContext = .{ - .b = b, - .c = c, - .q = b.ptrToInt(a.q, .i64), - .k = b.ptrToInt(a.k, .i64), - .v = b.ptrToInt(a.v, .i64), - .table = b.ptrToInt(a.table, .i64), - .indices = b.ptrToInt(a.indices, .i64), - .sink = b.ptrToInt(a.sink, .i64), - .out = b.ptrToInt(a.out, .i64), - .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), - .tid = tid, - .lane = tid.rem(32), - .warp = tid.div(32), - .seq = seq, - .head = head, - .first_row = first_row, - .token = token, - .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), - .split = b.blockIdx().z, - }; - t.loadQuery(); - // Split the selected-slot list, not the logical KV timeline. Repeated - // indices remain repeated softmax columns, including across splits. - const tiles = @divTrunc(c.topk + n - 1, n); - const lo = t.split.mul(tiles).div(c.splits); - const hi = t.split.add(1).mul(tiles).div(c.splits); - const row_base = t.warp.div(w).mul(16); - const dim_part = t.warp.rem(w).mul(d / w); - const row0 = t.lane.div(4); - const col0 = t.lane.rem(4).mul(2); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const of = d / w / 8; - const count = 4 + of; - var initial: [count]V = @splat(zero); - initial[0] = neg_inf; - initial[1] = neg_inf; - for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); - // A one-tile partition needs neither online rescaling nor a tile loop. - const result = if (c.splits == tiles) - advanceTile(t, d, w, n, dt, true, t.split.mul(n), initial) - else blk: { - var loop = b.openFor(lo, hi, 1, initial); - loop.yield(advanceTile(t, d, w, n, dt, false, loop.iv.mul(n), loop.carried)); - break :blk loop.results; - }; - if (c.splits > tiles) b.cpAsyncWaitAll(); - for (0..@as(usize, @intCast(@divExact(c.head_rows, 8)))) |h| { - const row = first_row.add(row_base).add(row0).add(h * 8); - var valid = b.openIf(row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows)))); - { - const m = result[h]; - const l = result[h + 2]; - var norm = b.select(l.gt(0), b.rcpApprox(l), zero); - if (c.splits == 1 and c.sink) { - const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row)).bitCast(.f32).mul(std.math.log2e); - const maximum = m.maximum(sink).maximum(-std.math.floatMax(f32)); - const weight = b.exp2Approx(m.sub(maximum)); - const denominator = l.mul(weight).add(b.exp2Approx(sink.sub(maximum))); - norm = weight.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)); - } - for (0..of) |i| { - var pair: [2]V = undefined; - for (&pair, 0..) |*value, j| { - const acc = b.vectorExtract(result[4 + i], @intCast(h * 2 + j), .f32); - value.* = if (c.splits > 1) acc else acc.mul(norm); - } - t.storePair(row, dim_part.add(i * 8).add(col0), pair[0], pair[1], dt); - } - if (c.splits > 1) { - var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); - { - t.store(row, b.constant(.i32, d), m); - t.store(row, b.constant(.i32, d + 1), l); - leader.close(); - } - } - valid.close(); - } - } - active.close(); - } -} - -fn advanceTile(t: TileContext, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType, comptime single: bool, key_base: V, previous: [4 + d / w / 8]V) [4 + d / w / 8]V { - const b = t.b; - const c = t.c; - const first_row = t.first_row; - const row_base = t.warp.div(w).mul(16); - const key_warps = @min(w, @min(n / 8, 4)); - const feature_parts = w / key_warps; - const feature_part = t.warp.div(key_warps).mul(d / feature_parts); - const key_part = t.warp.rem(key_warps).mul(n / key_warps); - const dim_part = t.warp.rem(w).mul(d / w); - const row0 = t.lane.div(4); - const col0 = t.lane.rem(4).mul(2); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const nf = n / key_warps / 8; - const halves: usize = @intCast(@divExact(c.head_rows, 8)); - const of = d / w / 8; - const count = 4 + of; - t.gather(key_base); - t.loadKV(t.k); - b.cpAsyncWaitAll(); - t.sync(); - var scores: [nf][4]V = @splat(@splat(zero)); - // Keep only MMA operands live in registers; Q remains in shared - // memory so D512 does not consume the entire per-thread register file. - for (0..d / feature_parts / 16) |ki| { - const qa = t.loadRows(t.shared, row_base, feature_part.add(ki * 16), d); - for (0..nf) |i| { - const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), feature_part.add(ki * 16).add(t.lane.div(8).rem(2).mul(8)), d))); - scores[i] = mma(b, dt, qa, kb, scores[i]); - } - } - if (feature_parts > 1) { - // Publishing partial dots proves every warp has finished reading K. - // Reuse its storage for V while the score warps combine the dots. - const bank = t.warp.div(key_warps); - for (0..nf) |i| for (0..halves * 2) |j| { - const register = t.warp.rem(key_warps).mul(@as(i64, nf) * c.head_rows * 8).add(i * @as(usize, @intCast(c.head_rows * 8)) + j * 32).add(t.lane); - const offset = bank.mul(c.head_rows * @as(i64, n)).add(register).mul(4); - b.storeWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{scores[i][j].bitCast(.i32)}); - }; - t.sync(); - t.loadKV(t.v); - } - const computesScores = if (feature_parts > 1) t.warp.lt(key_warps) else b.constant(.i1, 1); - var scoreStage = b.openIfElse(computesScores, @as([nf * 4 + 2]V, @splat(zero))); - { - if (feature_parts > 1) { - // Only the score warps combine dots and evaluate softmax. The - // remaining warps own output features and consume the shared P. - for (0..nf) |i| for (0..halves * 2) |j| { - const register = t.warp.rem(key_warps).mul(@as(i64, nf) * c.head_rows * 8).add(i * @as(usize, @intCast(c.head_rows * 8)) + j * 32).add(t.lane); - for (1..feature_parts) |peer| { - const offset = b.constant(.i32, peer * @as(usize, @intCast(c.head_rows)) * n).add(register).mul(4); - scores[i][j] = scores[i][j].add(b.loadWords(1, t.shared.add(c.pBase()).add(offset), .shared, .{})[0].bitCast(.f32)); - } - }; - } - var localMax: [2]V = @splat(neg_inf); - for (0..nf) |i| for (0..4) |j| { - const row = first_row.add(row_base).add(row0).add((j / 2) * 8); - const slot = key_part.add(i * 8).add(col0).add(j % 2); - const valid = row.lt(c.group).bitAnd(row.lt(first_row.add(c.head_rows))).bitAnd(t.validSlot(slot)); - scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); - localMax[j / 2] = localMax[j / 2].maximum(scores[i][j]); - }; - if (w == 1) { - for (&localMax) |*value| inline for (.{ 2, 1 }) |step| { - value.* = value.*.maximum(b.shuffleXor(value.*, step)); - }; - } else for (0..halves) |h| t.publishReduction(row0.add(h * 8), localMax[h], true); - var result: [nf * 4 + 2]V = undefined; - @memcpy(result[0 .. nf * 4], &flatten(nf, scores)); - @memcpy(result[nf * 4 ..], &localMax); - scoreStage.yieldThen(result); - } - scoreStage.yieldElse(@as([nf * 4 + 2]V, @splat(zero))); - for (0..nf) |i| scores[i] = scoreStage.results[i * 4 ..][0..4].*; - const local_max = scoreStage.results[nf * 4 ..][0..2].*; - if (w > 1) t.sync(); - if (feature_parts == 1) t.loadKV(t.v); - var m: [2]V = @splat(neg_inf); - var alpha: [2]V = @splat(zero); - var local_sum: [2]V = @splat(zero); - for (0..halves) |h| { - const tile_max = if (w == 1) local_max[h] else t.readReduction(key_warps, row0.add(h * 8), true); - m[h] = if (single) tile_max else previous[h].maximum(tile_max); - // Empty/fully masked row tiles have m=-inf and denominator 0. - alpha[h] = if (single) zero else b.exp2Approx(previous[h].sub(m[h].maximum(-std.math.floatMax(f32)))); - } - var probabilityStage = b.openIfElse(computesScores, @as([2]V, @splat(zero))); - { - for (0..nf) |i| { - for (0..4) |j| { - scores[i][j] = if (j / 2 < halves) b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))) else zero; - local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); - } - for (0..halves) |h| { - const row = row_base.add(row0).add(h * 8); - const col = key_part.add(i * 8).add(col0); - b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); - } - } - if (w == 1) { - for (&local_sum) |*value| inline for (.{ 2, 1 }) |step| { - value.* = value.*.add(b.shuffleXor(value.*, step)); - }; - } else for (0..halves) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); - probabilityStage.yieldThen(local_sum); - } - probabilityStage.yieldElse(@as([2]V, @splat(zero))); - local_sum = probabilityStage.results; - b.cpAsyncWaitAll(); - t.sync(); - var next: [count]V = undefined; - for (0..2) |h| { - next[h] = m[h]; - const tile_sum = if (h >= halves) zero else if (w == 1) local_sum[h] else t.readReduction(key_warps, row0.add(h * 8), false); - next[h + 2] = if (single) tile_sum else previous[h + 2].mul(alpha[h]).add(tile_sum); - } - var acc: [of][4]V = undefined; - for (0..of) |i| for (0..4) |j| { - acc[i][j] = if (single) zero else b.vectorExtract(previous[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); - }; - var packed_acc: [of]V = undefined; - for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); - var pv = b.openFor(0, n, 16, packed_acc); - { - const pa = t.loadRows(t.shared.add(c.pBase()), row_base, pv.iv, n); - for (0..of) |i| { - const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); - var prev: [4]V = undefined; - for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); - acc[i] = mma(b, dt, pa, vb, prev); - packed_acc[i] = packAcc(b, acc[i]); - } - pv.yield(packed_acc); - } - @memcpy(next[4..], &pv.results); - if (!single) t.sync(); - return next; -} - -pub const Merge = cute.Kernel(Config, .{ - .name = "paged_attention_cute_indexed_grouped_merge", - .inputs = &.{ "partials", "starts", "sink" }, - .outputs = &.{"out"}, - .run = merge, -}); - -// Lanes assigned to a feature own disjoint split partitions. Shuffles combine -// their softmax state, shortening the serial global-load chain at small batch. -fn merge(b: *B, c: Config) cute.FinishError!void { - if (c.merge_vector != 0) return switch (c.merge_vector) { - inline 4, 8, 16 => |features| switch (c.dtype) { - inline .bf16, .f16 => |dt| switch (c.merge_lanes) { - inline 8, 16, 32 => |parts| mergeVectorized(b, c, features, parts, dt), - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }; - return mergeScalar(b, c); -} - -// A lane owns a split and a contiguous feature vector. Reuse the score and -// weight for the entire vector, then reduce splits within each warp. -fn mergeVectorized(b: *B, c: Config, comptime features: usize, comptime parts: usize, comptime dt: cute.DType) cute.FinishError!void { - const a = try b.declareArgs(.{ - .partials = .{ .ptr = cute.DType.f32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .out = .{ .ptr = dt }, - }); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - const lane = b.threadIdx().x.rem(32); - const warp = b.threadIdx().x.div(32); - const featureLanes = 32 / parts; - const featureLane = lane.rem(featureLanes); - const partLane = lane.div(featureLanes); - const dim = b.blockIdx().z.mul(4 * featureLanes * features).add(warp.mul(featureLanes * features)).add(featureLane.mul(features)); - const slots = 128 / parts; - const query = token.to(.i64).mul(c.kv_heads * c.group).add(head.to(.i64)); - const out = b.ptrToInt(a.out, .i64).add(query.mul(c.head_dim).add(dim.to(.i64)).mul(2)); - const activeTokens = loadI32(b, b.ptrToInt(a.starts, .i64), b.constant(.i32, c.batch)); - const zero = b.constant(.f32, 0); - const negInf = b.constant(.f32, -std.math.inf(f32)); - var inactive = b.openIf(token.ge(activeTokens).bitAnd(partLane.eq(0))); - if (features == 4) { - b.storeWords(2, out, .global, @splat(b.constant(.i32, 0))); - } else for (0..features / 8) |chunk| b.storeWords(4, out.add(chunk * 16), .global, @splat(b.constant(.i32, 0))); - inactive.close(); - var active = b.openIf(token.lt(activeTokens)); - { - const base = b.ptrToInt(a.partials, .i64).add(query.mul(c.splits * c.partialWidth() * 4)); - const sink = if (c.sink) b.loadWords(1, b.ptrToInt(a.sink, .i64).add(head.to(.i64).mul(4)), .global, .{ .invariant = true })[0].bitCast(.f32).mul(std.math.log2e) else negInf; - const count: usize = @intCast(@divTrunc(c.splits + parts - 1, parts)); - var maxima: [slots]V = @splat(negInf); - var sums: [slots]V = @splat(zero); - var addresses: [slots]V = undefined; - var valid: [slots]V = undefined; - for (0..count) |i| { - const part = partLane.add(i * parts); - valid[i] = part.lt(c.splits); - addresses[i] = base.add(part.minimum(c.splits - 1).to(.i64).mul(c.partialWidth() * 4)); - const stats = b.loadWords(2, addresses[i].add(c.head_dim * 4), .global, .{ .invariant = true }); - sums[i] = b.select(valid[i], stats[1].bitCast(.f32), zero); - maxima[i] = b.select(valid[i].bitAnd(sums[i].gt(0)), stats[0].bitCast(.f32), negInf); - } - var reducedMaxima = maxima; - var stride: usize = 1; - while (stride < slots) : (stride *= 2) { - var i: usize = 0; - while (i < slots) : (i += 2 * stride) reducedMaxima[i] = reducedMaxima[i].maximum(reducedMaxima[i + stride]); - } - var maximum = reducedMaxima[0].maximum(sink); - var step: usize = featureLanes; - while (step < 32) : (step *= 2) maximum = maximum.maximum(b.shuffleXor(maximum, @intCast(step))); - maximum = maximum.maximum(-std.math.floatMax(f32)); - var weights: [slots]V = @splat(zero); - var denominator = zero; - for (0..count) |i| { - weights[i] = b.exp2Approx(maxima[i].sub(maximum)); - denominator = denominator.add(weights[i].mul(sums[i])); - } - if (c.sink) denominator = denominator.add(b.select(partLane.eq(0), b.exp2Approx(sink.sub(maximum)), zero)); - step = featureLanes; - while (step < 32) : (step *= 2) denominator = denominator.add(b.shuffleXor(denominator, @intCast(step))); - const norm = b.select(denominator.gt(0), b.rcpApprox(denominator), zero); - var values: [slots][features]V = @splat(@splat(zero)); - for (0..count) |i| for (0..features / 4) |chunk| { - const words = b.loadWords(4, addresses[i].add(dim.to(.i64).mul(4)).add(chunk * 16), .global, .{ .invariant = true }); - for (words, 0..) |word, j| values[i][chunk * 4 + j] = b.select(valid[i], weights[i].mul(word.bitCast(.f32)), zero); - }; - var result: [features]V = undefined; - for (&result, 0..) |*value, j| { - stride = 1; - while (stride < slots) : (stride *= 2) { - var i: usize = 0; - while (i < slots) : (i += 2 * stride) values[i][j] = values[i][j].add(values[i + stride][j]); - } - value.* = values[0][j]; - step = featureLanes; - while (step < 32) : (step *= 2) value.* = value.*.add(b.shuffleXor(value.*, @intCast(step))); - value.* = value.*.mul(norm); - } - var writer = b.openIf(partLane.eq(0)); - if (features == 4) { - b.storeWords(2, out, .global, .{ b.packHalf2(result[0], result[1], dt), b.packHalf2(result[2], result[3], dt) }); - } else for (0..features / 8) |chunk| { - var words: [4]V = undefined; - for (&words, 0..) |*word, j| word.* = b.packHalf2(result[chunk * 8 + j * 2], result[chunk * 8 + j * 2 + 1], dt); - b.storeWords(4, out.add(chunk * 16), .global, words); - } - writer.close(); - active.close(); - } -} - -fn mergeScalar(b: *B, c: Config) cute.FinishError!void { - std.debug.assert(c.splits > 0 and c.splits <= 128 and (c.merge_parts == 4 or c.merge_parts == 8 or c.merge_parts == 16 or c.merge_parts == 32)); - const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.tokens, c.kv_heads * c.group, c.splits, c.partialWidth() } } }, - .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, - .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, - .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - }); - const tid = b.threadIdx().x; - const cols = @divExact(32, c.merge_parts); - const part_lane = tid.rem(32).div(cols); - const dim = b.blockIdx().z.mul(4 * cols).add(tid.div(32).mul(cols)).add(tid.rem(cols)); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - const zero = b.constant(.f32, 0); - var inactive = b.openIf(token.ge(a.starts.get(.{c.batch})).bitAnd(part_lane.eq(0))); - a.out.set(.{ token, head, dim }, b.constant(c.dtype, 0)); - inactive.close(); - var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); - { - const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); - var maxima: [32]V = @splat(sink); - const groups = @divTrunc(c.splits + c.merge_parts - 1, c.merge_parts); - for (0..@as(usize, @intCast(groups))) |i| { - const part = part_lane.add(i * @as(usize, @intCast(c.merge_parts))); - maxima[i] = b.select(part.lt(c.splits), a.partials.get(.{ token, head, part.minimum(c.splits - 1), c.head_dim }), sink); - } - var max_stride: usize = 1; - while (max_stride < 32) : (max_stride *= 2) { - var i: usize = 0; - while (i < 32) : (i += 2 * max_stride) maxima[i] = maxima[i].maximum(maxima[i + max_stride]); - } - var m = maxima[0].maximum(sink); - var step = cols; - while (step < 32) : (step *= 2) m = m.maximum(b.shuffleXor(m, @intCast(step))); - m = m.maximum(-std.math.floatMax(f32)); - const chunk = @min(@as(i64, 32), @divTrunc(c.splits + c.merge_parts - 1, c.merge_parts)); - const sink_weight = if (c.sink) b.select(part_lane.eq(0), b.exp2Approx(sink.sub(m)), zero) else zero; - var sum = b.openFor(0, c.splits, chunk * c.merge_parts, .{ zero, sink_weight }); - { - var values: [32]V = @splat(zero); - var denominators: [32]V = @splat(zero); - for (0..@as(usize, @intCast(chunk))) |i| { - const part = sum.iv.add(part_lane).add(i * @as(usize, @intCast(c.merge_parts))); - const safe_part = part.minimum(c.splits - 1); - const weight = b.select(part.lt(c.splits), b.exp2Approx(a.partials.get(.{ token, head, safe_part, c.head_dim }).sub(m)), zero); - values[i] = weight.mul(a.partials.get(.{ token, head, safe_part, dim })); - denominators[i] = weight.mul(a.partials.get(.{ token, head, safe_part, c.head_dim + 1 })); - } - var stride: usize = 1; - while (stride < 32) : (stride *= 2) { - var i: usize = 0; - while (i < 32) : (i += 2 * stride) { - values[i] = values[i].add(values[i + stride]); - denominators[i] = denominators[i].add(denominators[i + stride]); - } - } - sum.yield(.{ sum.carried[0].add(values[0]), sum.carried[1].add(denominators[0]) }); - } - var numerator = sum.results[0]; - var denominator = sum.results[1]; - step = cols; - while (step < 32) : (step *= 2) { - numerator = numerator.add(b.shuffleXor(numerator, @intCast(step))); - denominator = denominator.add(b.shuffleXor(denominator, @intCast(step))); - } - var leader = b.openIf(part_lane.eq(0)); - a.out.set(.{ token, head, dim }, numerator.mul(b.select(denominator.gt(0), b.rcpApprox(denominator), zero)).to(c.dtype)); - leader.close(); - valid.close(); - } -} - -test "indexed grouped attention emits all supported head dimensions and dtypes" { - for ([_]i64{ 32, 64, 128, 256, 512 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| for ([_]i64{ 8, 16 }) |headRows| { - const cfg: Config = .{ - .head_dim = head_dim, - .head_rows = headRows, - .tokens = 7, - .batch = 3, - .kv_heads = 2, - .group = 19, - .pages_per_seq = 64, - .page_size = 16, - .topk = 37, - .dtype = dtype, - .kv_tile = if (head_dim == 512) 32 else 64, - .scale = 1 / @sqrt(@as(f32, @floatFromInt(head_dim))), - .sink = false, - }; - try std.testing.expectEqual(@as(i64, if (headRows == 8) 3 else 2), cfg.headTiles()); - try std.testing.expect(cfg.sharedBytes() <= 99 * 1024); - const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, if (dtype == .bf16) "mma.sync.aligned.m16n8k16.row.col.f32.bf16" else "mma.sync.aligned.m16n8k16.row.col.f32.f16") != null); - try std.testing.expectEqual(headRows == 16, std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x4") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cp.async") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); - }; -} - -test "indexed grouped attention emits split softmax and sink merge" { - for ([_]cute.DType{ .bf16, .f16 }) |dtype| { - const cfg: Config = .{ - .head_dim = 512, - .tokens = 7, - .batch = 3, - .kv_heads = 2, - .group = 19, - .pages_per_seq = 64, - .page_size = 16, - .topk = 257, - .dtype = dtype, - .kv_tile = 32, - .scale = 0.125, - .sink = true, - .splits = 3, - }; - const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - const merge_ir = try Merge.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); - defer std.testing.allocator.free(merge_ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x2.trans") != null); - try std.testing.expect(std.mem.indexOf(u8, merge_ir, "scf.for") != null); - } -} - -test "indexed grouped attention emits single warp D32 D64 D128" { - for ([_]i64{ 32, 64, 128 }) |head_dim| for ([_]cute.DType{ .bf16, .f16 }) |dtype| for ([_]i64{ 16, 32, 64 }) |tile| { - const cfg: Config = .{ - .head_dim = head_dim, - .tokens = 7, - .batch = 3, - .kv_heads = 2, - .group = 19, - .pages_per_seq = 64, - .page_size = 16, - .topk = 65, - .dtype = dtype, - .kv_tile = tile, - .warps = 1, - .scale = 0.125, - .sink = true, - .splits = 3, - }; - const ir = try Kernel.emit(std.testing.allocator, cfg, .{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "mma.sync.aligned.m16n8k16") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "ldmatrix.sync.aligned.m8n8.x2.trans") != null); - }; -} diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 9f640c8d..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -10,7 +10,7 @@ const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); +const triton = @import("triton/triton.zig"); const PagedAttention = @This(); diff --git a/zml/tensor.zig b/zml/tensor.zig index a09a5d38..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,7 +282,13 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, }