diff --git a/MODULE.bazel b/MODULE.bazel index 309bb188..d027db2d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -227,7 +227,7 @@ use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") git_override( module_name = "rules_zig", - commit = "b4ae496abfd61f880927dd01ea59823df7b42c29", + commit = "0570426cce634b110febba028bd692e0c4633b27", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index cdc0f1be..0cb72be2 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -35,7 +35,7 @@ pub const LlamaLM = struct { rms_norm_eps: f32, hf_rope_impl: bool = true, tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; pub const Options = struct { @@ -113,8 +113,8 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); @@ -227,8 +227,8 @@ pub const Llama = struct { tokens: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -274,8 +274,8 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -409,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 09d1f861..d850f1ea 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -25,7 +25,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, - backend: ?zml.attention.Backend = null, + backend: ?zml.attention.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -108,7 +108,7 @@ pub fn main() !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -126,7 +126,7 @@ pub fn main() !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.Backend.auto(platform); + const selected = zml.attention.attention.Backend.auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -192,8 +192,8 @@ pub fn main() !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -270,8 +270,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, }; const CompileModelResult = struct { @@ -436,7 +436,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, @@ -484,6 +484,10 @@ pub fn generateText( prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); } // Prepare for token-by-token generation, diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index ac5f8573..dc075e3c 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "arocc", remote = "https://github.com/zml/arocc.git", commit = "7e60c78a9660016e46d8be8907591b143ba2e700", - build_file = "//third_party/arocc:arocc.bazel", + build_file = "@zml//third_party/arocc:arocc.bazel", ) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 84781cb6..654620cc 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -4,8 +4,8 @@ load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): http_archive( name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", build_file = "//:third_party/flashattn/BUILD.bazel", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 2b3facf1..837fd36c 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "translate-c", remote = "https://github.com/zml/translate-c.git", commit = "5308ec6eba13a96dcaec452bffd0cc946384909a", - build_file = "//third_party/translate-c:translate-c.bazel", + build_file = "@zml//third_party/translate-c:translate-c.bazel", ) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index d55785ac..bf749a9a 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -83,6 +83,11 @@ pub fn main_016() !void { defer threaded.deinit(); const io = threaded.io(); + const path = try std.Io.Dir.realPathFileAlloc(.cwd(), io, ".", arena.allocator()); + + const execrootidx = std.mem.find(u8, path, "execroot/_main").?; + const workspace_dir = path[0 .. execrootidx + "execroot/_main".len]; + const args = try std.process.argsAlloc(arena.allocator()); defer std.process.argsFree(arena.allocator(), args); @@ -93,11 +98,10 @@ pub fn main_016() !void { var read_buffer: [8192]u8 = undefined; var reader = file.reader(io, &read_buffer); - const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); var replacer: StringReplaceReader = .{ .in = &reader.interface, .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", - .replace_by = build_workspace_directory, + .replace_by = workspace_dir, }; var write_buffer: [8192]u8 = undefined; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e8da1a27..30bb24d8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -20,8 +20,10 @@ cc_library( zig_library( name = "zml", srcs = [ + "attention.zig", "attention/attention.zig", "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -41,6 +43,7 @@ zig_library( "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", ], main = "zml.zig", diff --git a/zml/attention.zig b/zml/attention.zig new file mode 100644 index 00000000..1aa8001b --- /dev/null +++ b/zml/attention.zig @@ -0,0 +1,3 @@ +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); +pub const flashattn = @import("attention/flashattn.zig"); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..ad747f04 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,7 +1,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); -pub const flashattn = @import("flashattn.zig"); +const flashattn = @import("flashattn.zig"); const Attention = @This(); diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8673bc4e..111b7b1c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,16 +7,22 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); } } -pub fn register(platform: *zml.Platform) !void { +pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); } } @@ -52,6 +58,13 @@ const Buffer = struct { flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), ); } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } }; fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { @@ -121,7 +134,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: *zml.Platform) !void { + pub fn register(platform: *const zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } @@ -469,3 +482,957 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads, + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..deb63ec7 --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, + cuda_fa3, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.devices[0].pjrt_device; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, + + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), + }; +} diff --git a/zml/exe.zig b/zml/exe.zig index f5503d65..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,30 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } }; pub const Results = struct { diff --git a/zml/io.zig b/zml/io.zig index c39486bc..3763f445 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -120,6 +120,14 @@ pub const TensorStore = struct { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; @@ -451,8 +459,29 @@ pub const DirectMemoryWriter = struct { return std.Io.Writer.Error.WriteFailed; }; - if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { - defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; + self.offset += written; + if (is_last) { + // We're the last buffer, so we're waiting on it, otherwise the PJRT callback will point to an invalid context. + try self.wait_for_transfer_on_current_flip_flop(); + self.interface = .failing; + } else { + // Using the next context slot whenever it'll be available. + self.flip_flop ^= 1; + try self.wait_for_transfer_on_current_flip_flop(); + self.interface.end = 0; + self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { + log.err("unable to get a new buffer from the pool: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + } + return 0; + } + + fn wait_for_transfer_on_current_flip_flop(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + const ix: usize = @intCast(self.flip_flop); + const pjrt_api = self.memory.platform.pjrt_api; + if (self.events_contexts[ix]) |*ctx_previous| { + defer self.events_contexts[ix] = null; ctx_previous.event.waitUncancelable(self.io); defer ctx_previous.pjrt_event.deinit(pjrt_api); if (ctx_previous.err) |e| { @@ -464,19 +493,6 @@ pub const DirectMemoryWriter = struct { return std.Io.Writer.Error.WriteFailed; } } - - self.flip_flop ^= 1; - self.offset += written; - if (is_last) { - self.interface = .failing; - } else { - self.interface.end = 0; - self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { - log.err("unable to get a new buffer from the pool: {any}", .{err}); - return std.Io.Writer.Error.WriteFailed; - }; - } - return 0; } }; diff --git a/zml/nn.zig b/zml/nn.zig index cdc2b66b..a903c712 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,9 +27,8 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); + var y = x.dot(self.weight, self.tag); - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; @@ -91,12 +90,13 @@ pub const LayerNorm = struct { } }; -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); } /// Center and scale by the variance. @@ -143,7 +143,7 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, freq_base: f32 = 10_000, - scaling: Scaling = .default, + scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. @@ -153,10 +153,14 @@ pub const RopeOpts = struct { /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { - default: void, - custom: []const f32, + default: Default, llama3: Llama3, yarn: Yarn, + linear: Scaling.Linear, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; pub const Llama3 = struct { factor: f32, @@ -164,43 +168,94 @@ pub const RopeOpts = struct { low_freq_factor: f32, original_max_position_embeddings: u32, truncate: bool = true, + rope_theta: f32 = 10000, }; pub const Yarn = struct { beta_fast: f32 = 32.0, beta_slow: f32 = 1.0, - factor: f32, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, + rope_theta: f32, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, + }; + + pub const Linear = struct { + factor: f32, + rope_theta: f32 = 10000, }; /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; // autofix + _ = options; // autofix + if (source == .null) return .{ .default = .{} }; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; + return .{ .default = .{} }; } } + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + pub fn attentionScaling(scaling: Scaling) f32 { return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, else => 1.0, }; } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } }; }; @@ -285,9 +340,8 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 @@ -317,7 +371,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; + const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; diff --git a/zml/ops.zig b/zml/ops.zig index 757949cd..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1063,3 +1063,10 @@ fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); +} diff --git a/zml/slice.zig b/zml/slice.zig index d9fb9956..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -1,9 +1,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; pub fn isBytes(comptime T: type) bool { @@ -25,6 +25,7 @@ pub const Slice = struct { pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); return if (type_info.pointer.is_const) .{ .inner_data = .{ .immutable = bytes }, .shape = shape } diff --git a/zml/tensor.zig b/zml/tensor.zig index 957b4a44..a379db22 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1071,8 +1071,8 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {} tag, got {f}", .{ args, lhs.shape() }); - stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {} tag, got {f}", .{ args, rhs.shape() }); + stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); + stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -287,10 +287,13 @@ pub fn testLayer( layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, - comptime name: []const u8, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), opts: CompareOpts, ) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -301,13 +304,14 @@ pub fn testLayer( index: usize = 0, }; + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(name ++ ".in"); + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -322,9 +326,10 @@ pub fn testLayer( const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; - const store_output = activation_store.withPrefix(name ++ ".out"); - const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -365,7 +370,7 @@ pub fn testLayer( try reader.interface.readSliceAll(expected_slice.data()); break :b expected_slice; }; - defer allocator.free(expected_slice.data()); + defer expected_slice.free(allocator); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -375,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); failed = true; continue; }, diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 37252e74..0ab7e940 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -12,7 +12,7 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention/attention.zig"); +pub const attention = @import("attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); @@ -42,6 +42,7 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This());