From 0e0570fa51914e16339ee3ff348b3f267b49f088 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 19 Jan 2026 17:33:52 +0100 Subject: [PATCH 01/27] workspace: fix to allow use of ZML in monorepo --- MODULE.bazel | 2 +- third_party/arocc/repo.bzl | 2 +- third_party/translate-c/repo.bzl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 309bb188..d027db2d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -227,7 +227,7 @@ use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") git_override( module_name = "rules_zig", - commit = "b4ae496abfd61f880927dd01ea59823df7b42c29", + commit = "0570426cce634b110febba028bd692e0c4633b27", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index ac5f8573..dc075e3c 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "arocc", remote = "https://github.com/zml/arocc.git", commit = "7e60c78a9660016e46d8be8907591b143ba2e700", - build_file = "//third_party/arocc:arocc.bazel", + build_file = "@zml//third_party/arocc:arocc.bazel", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 2b3facf1..837fd36c 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "translate-c", remote = "https://github.com/zml/translate-c.git", commit = "5308ec6eba13a96dcaec452bffd0cc946384909a", - build_file = "//third_party/translate-c:translate-c.bazel", + build_file = "@zml//third_party/translate-c:translate-c.bazel", ) From b0c0527d498362090b6b73e24f93cd6f337f6dea Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 21 Jan 2026 09:50:04 +0100 Subject: [PATCH 02/27] third_party/zls: fix so that we can use it outside of zml --- third_party/zls/workspace_printer.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index d55785ac..bf749a9a 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -83,6 +83,11 @@ pub fn main_016() !void { defer threaded.deinit(); const io = threaded.io(); + const path = try std.Io.Dir.realPathFileAlloc(.cwd(), io, ".", arena.allocator()); + + const execrootidx = std.mem.find(u8, path, "execroot/_main").?; + const workspace_dir = path[0 .. execrootidx + "execroot/_main".len]; + const args = try std.process.argsAlloc(arena.allocator()); defer std.process.argsFree(arena.allocator(), args); @@ -93,11 +98,10 @@ pub fn main_016() !void { var read_buffer: [8192]u8 = undefined; var reader = file.reader(io, &read_buffer); - const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); var replacer: StringReplaceReader = .{ .in = &reader.interface, .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", - .replace_by = build_workspace_directory, + .replace_by = workspace_dir, }; var write_buffer: [8192]u8 = undefined; From f5b95c3c134394ed2096722c9b74c654ed09e194 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 Jan 2026 14:28:01 +0100 Subject: [PATCH 03/27] zml: added some paged attention bindings --- zml/BUILD.bazel | 2 + zml/attention.zig | 3 + zml/attention/attention.zig | 2 +- zml/attention/flashattn.zig | 486 +++++++++++++++++++++++++++++- zml/attention/paged_attention.zig | 55 ++++ zml/zml.zig | 2 +- 6 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 zml/attention.zig create mode 100644 zml/attention/paged_attention.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e8da1a27..d0c88e6c 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -20,8 +20,10 @@ cc_library( zig_library( name = "zml", srcs = [ + "attention.zig", "attention/attention.zig", "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", "constants.zig", "dtype.zig", diff --git a/zml/attention.zig b/zml/attention.zig new file mode 100644 index 00000000..1aa8001b --- /dev/null +++ b/zml/attention.zig @@ -0,0 +1,3 @@ +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); +pub const flashattn = @import("attention/flashattn.zig"); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..ad747f04 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,7 +1,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); -pub const flashattn = @import("flashattn.zig"); +const flashattn = @import("flashattn.zig"); const Attention = @This(); diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8673bc4e..ebf87ca9 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -17,6 +17,8 @@ pub fn register(platform: *zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); } } @@ -121,8 +123,8 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: *zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + pub fn register(platform: zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "host", T.run, .{ .command_buffer_compatible = true }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -469,3 +471,483 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_seqlen_q: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + max_seqlen_q: usize, + decode_offset: ?zml.Tensor = null, + + //pub fn init(options: Options) zml.ShapeOf(Context) { + // const max_seqlen_k = switch (options) { + // inline else => |v| v.max_seqlen_k, + // }; + // const decode_offset: ?zml.Shape = switch (options) { + // .mixed => zml.Shape.init(.{1}, .i32), + // .decode => null, + // }; + // return .{ + // .max_seqlen_k = max_seqlen_k, + // .decode_offset = decode_offset, + // }; + //} + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const max_seqlen_q = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.options.max_seqlen_q, + else => 1, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .max_seqlen_q = max_seqlen_q, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = context.max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.block_table, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = context.max_seqlen_q, + .num_heads = num_heads, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.block_table_prefill, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = context.max_seqlen_q, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.block_table_decode, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = 1, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct {}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..acf99b5a --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,55 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), + }; +} diff --git a/zml/zml.zig b/zml/zml.zig index 37252e74..63066971 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -12,7 +12,7 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention/attention.zig"); +pub const attention = @import("attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); From 722ad5254bff12c545ec8556660583148aa80f0b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 23 Jan 2026 20:30:30 +0000 Subject: [PATCH 04/27] wip it works --- examples/llama/llama.zig | 18 +++---- examples/llama/main.zig | 20 +++++--- zml/attention/flashattn.zig | 100 +++++++++++++++++++++++------------- zml/slice.zig | 3 +- 4 files changed, 87 insertions(+), 54 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index cdc0f1be..a4594a48 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -113,8 +113,8 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); @@ -227,8 +227,8 @@ pub const Llama = struct { tokens: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -274,8 +274,8 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -409,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 09d1f861..d850f1ea 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -25,7 +25,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, - backend: ?zml.attention.Backend = null, + backend: ?zml.attention.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -108,7 +108,7 @@ pub fn main() !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -126,7 +126,7 @@ pub fn main() !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.Backend.auto(platform); + const selected = zml.attention.attention.Backend.auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -192,8 +192,8 @@ pub fn main() !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -270,8 +270,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, }; const CompileModelResult = struct { @@ -436,7 +436,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, @@ -484,6 +484,10 @@ pub fn generateText( prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); } // Prepare for token-by-token generation, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index ebf87ca9..fa758b2b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -54,6 +54,13 @@ const Buffer = struct { flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), ); } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } }; fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { @@ -124,7 +131,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "host", T.run, .{ .command_buffer_compatible = true }); + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -561,6 +568,16 @@ pub const paged_fa2 = struct { }; } + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); var allocation_size: usize = 0; @@ -574,7 +591,6 @@ pub const paged_fa2 = struct { batch_size_decode: usize, max_num_pages: usize, max_seqlen_k: usize, - max_seqlen_q: usize, max_token_count: usize, num_heads: usize, head_dim: usize, @@ -632,6 +648,17 @@ pub const paged_fa2 = struct { }; } + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); var allocation_size: usize = 0; @@ -643,7 +670,6 @@ pub const paged_fa2 = struct { pub const Context = struct { max_seqlen_k: usize, - max_seqlen_q: usize, decode_offset: ?zml.Tensor = null, //pub fn init(options: Options) zml.ShapeOf(Context) { @@ -668,15 +694,11 @@ pub const paged_fa2 = struct { const max_seqlen_k = switch (parameters) { inline else => |v| v.options.max_seqlen_k, }; - const max_seqlen_q = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.options.max_seqlen_q, - else => 1, - }; const decode_offset = switch (parameters) { .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), .decode => null, }; - return .{ .max_seqlen_k = max_seqlen_k, .max_seqlen_q = max_seqlen_q, .decode_offset = decode_offset }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; } }; @@ -687,23 +709,26 @@ pub const paged_fa2 = struct { const register = Wrapped.register; const run = Wrapped.run; - fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { const head_dim = q.shape.dim(2); break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); @@ -711,7 +736,6 @@ pub const paged_fa2 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; @@ -735,7 +759,7 @@ pub const paged_fa2 = struct { &paged_v.toFlashattnTensor(), &o.toFlashattnTensor(), &cu_seqlens_q.toFlashattnTensor(), - null, + &cu_seqlens_k.toFlashattnTensor(), &seqused_k.toFlashattnTensor(), &block_table.toFlashattnTensor(), &softmax_lse.toFlashattnTensor(), @@ -757,21 +781,22 @@ pub const paged_fa2 = struct { const register = Wrapped.register; const run = Wrapped.run; - fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { @@ -781,7 +806,6 @@ pub const paged_fa2 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; @@ -789,7 +813,7 @@ pub const paged_fa2 = struct { const stream = call_frame.api.stream(ctx); const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, + .max_seqlen_q = 1, .max_seqlen_k = max_seqlen_k, .is_causal = is_causal, .softmax_scale = softmax_scale, @@ -805,7 +829,7 @@ pub const paged_fa2 = struct { &paged_v.toFlashattnTensor(), &o.toFlashattnTensor(), &cu_seqlens_q.toFlashattnTensor(), - null, + &cu_seqlens_k.toFlashattnTensor(), &seqused_k.toFlashattnTensor(), &block_table.toFlashattnTensor(), &softmax_lse.toFlashattnTensor(), @@ -829,13 +853,14 @@ pub const paged_fa2 = struct { .decode => |decode_parameters| b: { const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); const original_tot = q.dim(.b); const num_heads = q.dim(.h); const num_heads_k = k_cache.dim(.h); const head_size = q.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = context.max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; var q2 = q; if (seqlenq_ngroups_swapped) { q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); @@ -847,9 +872,10 @@ pub const paged_fa2 = struct { q2, k_cache, v_cache, - decode_parameters.block_table, decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, decode_parameters.seqused_k, + decode_parameters.block_table, softmax_lse, softmax_lse_accum, decode_parameters.metadata.out_accum, @@ -859,7 +885,6 @@ pub const paged_fa2 = struct { .{ .is_causal = true, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = context.max_seqlen_q, .num_heads = num_heads, }, .{ .has_side_effect = false }, @@ -874,25 +899,27 @@ pub const paged_fa2 = struct { .mixed => |mixed_parameters| b: { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); var o = zml.ops.customCall( Prefill.custom_call_name, .{ q, k_cache, v_cache, - mixed_parameters.block_table_prefill, mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, softmax_lse_prefill, softmax_lse_accum_prefill, mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, layer_index, }, .{q.shape()}, .{ - .is_causal = true, + .is_causal = false, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = context.max_seqlen_q, .num_heads = q.dim(.h), }, .{ .has_side_effect = false }, @@ -901,6 +928,7 @@ pub const paged_fa2 = struct { const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); const original_tot = q_decode.dim(.b); @@ -919,9 +947,10 @@ pub const paged_fa2 = struct { q_decode, k_cache, v_cache, - mixed_parameters.block_table_decode, mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, mixed_parameters.metadata.out_accum, @@ -931,7 +960,6 @@ pub const paged_fa2 = struct { .{ .is_causal = true, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = 1, .num_heads = q.dim(.h), }, .{ .has_side_effect = false }, diff --git a/zml/slice.zig b/zml/slice.zig index d9fb9956..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -1,9 +1,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; pub fn isBytes(comptime T: type) bool { @@ -25,6 +25,7 @@ pub const Slice = struct { pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); return if (type_info.pointer.is_const) .{ .inner_data = .{ .immutable = bytes }, .shape = shape } From aaf139dfe49c2588efeb065aa1783bd8fa2addcb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:15 +0000 Subject: [PATCH 05/27] wip: argument baking --- zml/exe.zig | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/zml/exe.zig b/zml/exe.zig index f5503d65..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,30 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } }; pub const Results = struct { From b1f12e194a7563cc7aadc29edd775376f72fcdd7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:54 +0000 Subject: [PATCH 06/27] Added paged attention FA3 --- zml/attention/flashattn.zig | 489 +++++++++++++++++++++++++++++- zml/attention/paged_attention.zig | 8 + 2 files changed, 481 insertions(+), 16 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index fa758b2b..ec402dc7 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -19,6 +19,8 @@ pub fn register(platform: *zml.Platform) !void { try fa3.register(platform); try paged_fa2.Decode.register(platform); try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); } } @@ -131,7 +133,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -672,20 +674,6 @@ pub const paged_fa2 = struct { max_seqlen_k: usize, decode_offset: ?zml.Tensor = null, - //pub fn init(options: Options) zml.ShapeOf(Context) { - // const max_seqlen_k = switch (options) { - // inline else => |v| v.max_seqlen_k, - // }; - // const decode_offset: ?zml.Shape = switch (options) { - // .mixed => zml.Shape.init(.{1}, .i32), - // .decode => null, - // }; - // return .{ - // .max_seqlen_k = max_seqlen_k, - // .decode_offset = decode_offset, - // }; - //} - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { _ = num_heads; // autofix _ = num_kv_heads; // autofix @@ -978,4 +966,473 @@ pub const paged_fa2 = struct { } }; -pub const paged_fa3 = struct {}; +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + scheduler_metadata_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index acf99b5a..767ac8ee 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -7,10 +7,12 @@ const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, + cuda_fa3, }; pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, pub fn isPrefill(self: Options) bool { return switch (self) { @@ -21,16 +23,19 @@ pub const Options = union(Backend) { pub const Parameters = union(Backend) { cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, }; } pub fn options(self: Parameters) Options { return switch (self) { .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, }; } }; @@ -38,10 +43,12 @@ pub const Parameters = union(Backend) { /// Internal state that can be used inside the model code. It's derived from Parameters and Option. pub const Context = union(Backend) { cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, }; } }; @@ -51,5 +58,6 @@ pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k _ = v; // autofix return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), }; } From bd2bfdde9ac2f24434d7b1125280c30f782f3c08 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:14 +0000 Subject: [PATCH 07/27] zml: added simple tracer support back --- zml/BUILD.bazel | 1 + zml/tracer.zig | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ zml/zml.zig | 1 + 3 files changed, 79 insertions(+) create mode 100644 zml/tracer.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index d0c88e6c..30bb24d8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -43,6 +43,7 @@ zig_library( "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", ], main = "zml.zig", diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 63066971..0ab7e940 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -42,6 +42,7 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This()); From 8ab0d04b2854fe3ff31eafc2bbd383be7e9320aa Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:36 +0000 Subject: [PATCH 08/27] flashattn lib override --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 84781cb6..a500ec9f 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - http_archive( - name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - build_file = "//:third_party/flashattn/BUILD.bazel", - ) + #http_archive( + # name = "flashattn", + # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", + # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + # build_file = "//:third_party/flashattn/BUILD.bazel", + #) - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From ac4a0e80604ce18dbdf1d8bae05523f2e20f6d72 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 10:41:10 +0000 Subject: [PATCH 09/27] small fixes --- zml/attention/flashattn.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index ec402dc7..4f4aa26e 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1051,7 +1051,7 @@ pub const paged_fa3 = struct { pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { return .{ - .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), }; } @@ -1416,8 +1416,8 @@ pub const paged_fa3 = struct { mixed_parameters.block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - scheduler_metadata_decode, mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, layer_index, }, .{q_decode.shape()}, From fbc7965ce88f870f9f78b2fb2ef81f22637ca76b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 16:17:36 +0000 Subject: [PATCH 10/27] third_party/flashattn: bump libflashattn.so --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index a500ec9f..654620cc 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - #http_archive( - # name = "flashattn", - # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - # build_file = "//:third_party/flashattn/BUILD.bazel", - #) + http_archive( + name = "flashattn", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + build_file = "//:third_party/flashattn/BUILD.bazel", + ) - new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From a4cee2db071c95b3c9fa5f69ef12ab5dc8c04d07 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:07 +0000 Subject: [PATCH 11/27] zml/attention: fix wrong causal option --- zml/attention/flashattn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 4f4aa26e..f7d1f38b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -906,7 +906,7 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = false, + .is_causal = true, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), }, From 955cd89af12860113cb0dbb09e9aaf6b7adb1f91 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:33 +0000 Subject: [PATCH 12/27] zml/nn: removed commented code --- zml/nn.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index cdc2b66b..c53b840b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -29,7 +29,6 @@ pub const Linear = struct { pub fn forward(self: Linear, x: Tensor) Tensor { var y = x.dot(self.weight.convert(x.dtype()), self.tag); - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; From 26d223ca52c599cbf27fd0e8eff30321061e7a92 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:08:05 +0000 Subject: [PATCH 13/27] zml/nn: Use higher precision for rmsNorm --- zml/nn.zig | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index c53b840b..0a53de0e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -90,12 +90,13 @@ pub const LayerNorm = struct { } }; -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); } /// Center and scale by the variance. From a3fe30791a60a42e17505614237135c1af439386 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:18:41 +0000 Subject: [PATCH 14/27] zml/io: added TensorStore.hasKey() to check key existence It's a bit hacky since it does an array traversal to see if there is a match. --- zml/io.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zml/io.zig b/zml/io.zig index c39486bc..7a33f432 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -120,6 +120,14 @@ pub const TensorStore = struct { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; From e668996fea3fbfc06c23d1646cfe80e31f42e948 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:21:53 +0000 Subject: [PATCH 15/27] zml/attention: Re-introduced allocationSize --- zml/attention/flashattn.zig | 98 +++++++++++++++---------------- zml/attention/paged_attention.zig | 6 ++ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f7d1f38b..354252e4 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -507,10 +507,10 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -549,13 +549,13 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -580,10 +580,9 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -623,17 +622,16 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -661,11 +659,10 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; @@ -992,10 +989,10 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -1034,13 +1031,13 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1065,10 +1062,9 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -1108,17 +1104,16 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1146,11 +1141,10 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 767ac8ee..f07eb289 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -38,6 +38,12 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, }; } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } }; /// Internal state that can be used inside the model code. It's derived from Parameters and Option. From 19a07b87fb2aa3c4437c26f9d2f018bf516f4ee4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:22:36 +0000 Subject: [PATCH 16/27] zml/nn: Improved Yarn logic --- zml/nn.zig | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 0a53de0e..f6a9617f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -169,35 +169,64 @@ pub const RopeOpts = struct { pub const Yarn = struct { beta_fast: f32 = 32.0, beta_slow: f32 = 1.0, - factor: f32, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, + rope_theta: f32, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, }; /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; // autofix + _ = options; // autofix + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + pub fn attentionScaling(scaling: Scaling) f32 { return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, else => 1.0, }; } @@ -317,7 +346,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; + const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; From 8b453c71393a5a804f71971c2e32f279ea41342b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:05:11 +0000 Subject: [PATCH 17/27] zml/testing: Made testlayer take a runtime name --- zml/testing.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..6fda9241 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -287,10 +287,13 @@ pub fn testLayer( layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, - comptime name: []const u8, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), opts: CompareOpts, ) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -301,13 +304,14 @@ pub fn testLayer( index: usize = 0, }; + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(name ++ ".in"); + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -322,9 +326,10 @@ pub fn testLayer( const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; - const store_output = activation_store.withPrefix(name ++ ".out"); - const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -365,7 +370,7 @@ pub fn testLayer( try reader.interface.readSliceAll(expected_slice.data()); break :b expected_slice; }; - defer allocator.free(expected_slice.data()); + defer expected_slice.free(allocator); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -375,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + try w.print("{s}.{d} doesn't match !", .{ output_name, i }); failed = true; continue; }, From 833ef78715c6323454136b45dcd6506ac74549b2 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:10:11 +0000 Subject: [PATCH 18/27] zml: added ops.scaledDot() It implements a scaled dot product --- zml/ops.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zml/ops.zig b/zml/ops.zig index 757949cd..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1063,3 +1063,10 @@ fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); +} From 4505fc3703c1396500633cb1dab3de34341c56b8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:29:31 +0000 Subject: [PATCH 19/27] zml: added some convenience to paged_attention --- zml/attention/paged_attention.zig | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index f07eb289..77f49125 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); @@ -8,12 +9,82 @@ const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, cuda_fa3, + + pub fn auto(platform: zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } }; pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + pub fn isPrefill(self: Options) bool { return switch (self) { inline else => |v| v.isPrefill(), From 64885473354e5eabc41b366b1b4125182e334b2e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 11 Feb 2026 21:01:42 +0100 Subject: [PATCH 20/27] zml/io: Fix race condition for last flipflop transfer --- zml/io.zig | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 7a33f432..3763f445 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -459,8 +459,29 @@ pub const DirectMemoryWriter = struct { return std.Io.Writer.Error.WriteFailed; }; - if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { - defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; + self.offset += written; + if (is_last) { + // We're the last buffer, so we're waiting on it, otherwise the PJRT callback will point to an invalid context. + try self.wait_for_transfer_on_current_flip_flop(); + self.interface = .failing; + } else { + // Using the next context slot whenever it'll be available. + self.flip_flop ^= 1; + try self.wait_for_transfer_on_current_flip_flop(); + self.interface.end = 0; + self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { + log.err("unable to get a new buffer from the pool: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + } + return 0; + } + + fn wait_for_transfer_on_current_flip_flop(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + const ix: usize = @intCast(self.flip_flop); + const pjrt_api = self.memory.platform.pjrt_api; + if (self.events_contexts[ix]) |*ctx_previous| { + defer self.events_contexts[ix] = null; ctx_previous.event.waitUncancelable(self.io); defer ctx_previous.pjrt_event.deinit(pjrt_api); if (ctx_previous.err) |e| { @@ -472,19 +493,6 @@ pub const DirectMemoryWriter = struct { return std.Io.Writer.Error.WriteFailed; } } - - self.flip_flop ^= 1; - self.offset += written; - if (is_last) { - self.interface = .failing; - } else { - self.interface.end = 0; - self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { - log.err("unable to get a new buffer from the pool: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - } - return 0; } }; From d3f7f132c8d0f3e29f1c41540f73849d25f0e653 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:47:17 +0000 Subject: [PATCH 21/27] zml/tensor: fix formatting in `dot` --- zml/tensor.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 957b4a44..a379db22 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1071,8 +1071,8 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {} tag, got {f}", .{ args, lhs.shape() }); - stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {} tag, got {f}", .{ args, rhs.shape() }); + stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); + stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); From 3c3a6f5b1b9a60f97f855a5e19894391bc573a57 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:50:17 +0000 Subject: [PATCH 22/27] zml/attention: update to latest zml/v2 changes --- zml/attention/flashattn.zig | 4 ++-- zml/attention/paged_attention.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 354252e4..0a11629b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -13,7 +13,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { } } -pub fn register(platform: *zml.Platform) !void { +pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); @@ -132,7 +132,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: zml.Platform) !void { + pub fn register(platform: *const zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 77f49125..29f05876 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -10,7 +10,7 @@ pub const Backend = enum { cuda_fa2, cuda_fa3, - pub fn auto(platform: zml.Platform) Backend { + pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; From 74832f343b2b81cda61511e63b163fe8a48693cd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:11 +0000 Subject: [PATCH 23/27] zml/nn: added common `rope_theta` parameter to rope variants --- examples/llama/llama.zig | 2 +- zml/nn.zig | 34 +++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index a4594a48..0cb72be2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -35,7 +35,7 @@ pub const LlamaLM = struct { rms_norm_eps: f32, hf_rope_impl: bool = true, tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; pub const Options = struct { diff --git a/zml/nn.zig b/zml/nn.zig index f6a9617f..167925c6 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -143,7 +143,7 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, freq_base: f32 = 10_000, - scaling: Scaling = .default, + scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. @@ -153,10 +153,14 @@ pub const RopeOpts = struct { /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { - default: void, - custom: []const f32, + default: Default, llama3: Llama3, yarn: Yarn, + linear: Scaling.Linear, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; pub const Llama3 = struct { factor: f32, @@ -164,6 +168,7 @@ pub const RopeOpts = struct { low_freq_factor: f32, original_max_position_embeddings: u32, truncate: bool = true, + rope_theta: f32 = 10000, }; pub const Yarn = struct { @@ -179,6 +184,11 @@ pub const RopeOpts = struct { attention_factor: ?f32 = null, }; + pub const Linear = struct { + factor: f32, + rope_theta: f32, + }; + /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -191,7 +201,7 @@ pub const RopeOpts = struct { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { _ = allocator; // autofix _ = options; // autofix - if (source == .null) return .default; + if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; @@ -203,9 +213,11 @@ pub const RopeOpts = struct { return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; + return .{ .default = .{} }; } } @@ -230,6 +242,13 @@ pub const RopeOpts = struct { else => 1.0, }; } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } }; }; @@ -314,10 +333,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); - }, + .linear => @panic("Not implemented"), .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From 6391e900393a6a13e3ab56af69a0eeb9856a1682 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:56 +0000 Subject: [PATCH 24/27] zml/testing: formatting fix --- zml/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index 6fda9241..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -380,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ output_name, i }); + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); failed = true; continue; }, From db99a312c9ee1e3b7d9aa3264e1a61efa13dd336 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:21 +0000 Subject: [PATCH 25/27] zml/attention: added options for paged_attention --- zml/attention/flashattn.zig | 26 ++++++++++++++++---------- zml/attention/paged_attention.zig | 13 +++++++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 0a11629b..111b7b1c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,6 +7,8 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); @@ -829,7 +831,7 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -837,7 +839,7 @@ pub const paged_fa2 = struct { const o = switch (parameters) { .decode => |decode_parameters| b: { const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); const original_tot = q.dim(.b); @@ -845,7 +847,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; var q2 = q; if (seqlenq_ngroups_swapped) { q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); @@ -868,9 +870,10 @@ pub const paged_fa2 = struct { }, .{q2.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads, + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -883,7 +886,7 @@ pub const paged_fa2 = struct { }, .mixed => |mixed_parameters| b: { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); var o = zml.ops.customCall( Prefill.custom_call_name, @@ -903,16 +906,17 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); @@ -921,7 +925,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q_decode.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); } @@ -943,9 +947,10 @@ pub const paged_fa2 = struct { }, .{q_decode.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -1329,7 +1334,8 @@ pub const paged_fa3 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 29f05876..deb63ec7 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -13,7 +13,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const first_device = platform.devices[0].pjrt_device; if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { if (std.mem.eql(u8, cc, "9.0")) { @@ -130,11 +130,16 @@ pub const Context = union(Backend) { } }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { _ = k; // autofix _ = v; // autofix return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), }; } From ea676abb595704f41c91643cba3491b4ef91a43f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:48 +0000 Subject: [PATCH 26/27] zml/nn: added default value for rope_theta + linear scaling --- zml/nn.zig | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 167925c6..ee79e9a9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -186,7 +186,7 @@ pub const RopeOpts = struct { pub const Linear = struct { factor: f32, - rope_theta: f32, + rope_theta: f32 = 10000, }; /// Read a Rope scaling config from HF config.json format. @@ -249,6 +249,13 @@ pub const RopeOpts = struct { inline else => |*s| s.rope_theta = rope_theta, }; } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } }; }; @@ -333,7 +340,9 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .linear => @panic("Not implemented"), + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; + }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From 11e6b2a85987e186bb5ac760b011c9be841678a0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:06:06 +0000 Subject: [PATCH 27/27] zml/nn: remove useless convert --- zml/nn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index ee79e9a9..a903c712 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,7 +27,7 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); + var y = x.dot(self.weight, self.tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; }