diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index fa567455..df874ce6 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -35,7 +35,7 @@ pub const LlamaLM = struct { rms_norm_eps: f32, hf_rope_impl: bool = true, tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; pub const Options = struct { @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters, .{}); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index b23e6601..ee685a8c 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -103,7 +103,7 @@ pub fn main(init: std.process.Init) !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -121,7 +121,7 @@ pub fn main(init: std.process.Init) !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.Backend.auto(platform); + const selected: zml.attention.Backend = .auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -483,6 +483,10 @@ pub fn generateText( prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); } // Prepare for token-by-token generation, diff --git a/examples/voxtral/BUILD.bazel b/examples/voxtral/BUILD.bazel new file mode 100644 index 00000000..7abf520f --- /dev/null +++ b/examples/voxtral/BUILD.bazel @@ -0,0 +1,43 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "voxtral", + main = "main.zig", + srcs = [ + "voxtral.zig", + "mel_spectrogram.zig", + "config.zig", + "encoder.zig", + "decoder.zig", + "common.zig", + "terminalwave/terminalwave.zig" + ], + extra_srcs = [ + "//examples/voxtral/assets:voxtral_mel_filter", + ], + deps = ["//zml"], +) + +zig_binary( + name = "voxtral_test", + main = "test.zig", + srcs = [ + "voxtral.zig", + "mel_spectrogram.zig", + "config.zig", + "encoder.zig", + "decoder.zig", + "common.zig", + "terminalwave/terminalwave.zig" + ], + extra_srcs = [ + "//examples/voxtral/assets:voxtral_mel_filter", + ], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":voxtral_test"], +) diff --git a/examples/voxtral/assets/BUILD.bazel b/examples/voxtral/assets/BUILD.bazel new file mode 100644 index 00000000..c60a233b --- /dev/null +++ b/examples/voxtral/assets/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "voxtral_mel_filter", + srcs = ["voxtral_mel_filter.data"], + visibility = ["//visibility:public"], +) diff --git a/examples/voxtral/assets/voxtral_mel_filter.data b/examples/voxtral/assets/voxtral_mel_filter.data new file mode 100644 index 00000000..7e8a85eb Binary files /dev/null and b/examples/voxtral/assets/voxtral_mel_filter.data differ diff --git a/examples/voxtral/common.zig b/examples/voxtral/common.zig new file mode 100644 index 00000000..00479949 --- /dev/null +++ b/examples/voxtral/common.zig @@ -0,0 +1,304 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; +const Shape = zml.Shape; + +const cfg = @import("config.zig"); + +/// Recursively deinit all Buffer fields in a Bufferized struct. +/// For models with allocated slices (e.g. layers), the caller must also free the slice. +pub fn deinitBufferized(bufferized: anytype) void { + zml.meta.visit((struct { + fn cb(_: void, buf: *zml.Buffer) void { + buf.deinit(); + } + }).cb, {}, bufferized); +} + +pub fn linear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init( + store.createTensorWithTags("weight", .{ .dout, .d }), + store.maybeCreateTensorWithTags("bias", .{.dout}), + .d, + ); +} + +pub fn rmsNorm(x: Tensor, weight: Tensor, eps: f32) Tensor { + return zml.nn.rmsNorm(x, .d, eps).mul(weight.broad(x.shape())); +} + +pub fn loadModel( + comptime T: type, + self: *const T, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + comptime label: []const u8, +) !zml.Bufferized(T) { + progress.increaseEstimatedTotalItems(store.view().count()); + + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + log.info("Loaded " ++ label ++ " weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + stdx.fmt.fmtDuration(took), + total_bytes / @as(usize, @intCast(took.toNanoseconds())) * std.time.ns_per_s, + }); + } + + return zml.io.load(T, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); +} + +/// Unified self-attention with KV cache. +/// When `circular_buffer` is true, the KV cache uses a circular (ring) buffer for unbounded +/// sliding window support: keys/values are written at position `pos_index % cache_size`, +/// then reordered into temporal order before attention. +/// When false, the cache uses sequential indexing. +pub fn SelfAttention(comptime circular_buffer: bool) type { + return struct { + wq: zml.nn.Linear, + wk: zml.nn.Linear, + wv: zml.nn.Linear, + wo: zml.nn.Linear, + + const Self = @This(); + + pub fn init(store: zml.io.TensorStore.View) Self { + return .{ + .wq = linear(store.withPrefix("wq")), + .wk = linear(store.withPrefix("wk")), + .wv = linear(store.withPrefix("wv")), + .wo = linear(store.withPrefix("wo")), + }; + } + + pub fn unload(self: *zml.Bufferized(Self)) void { + deinitBufferized(self); + } + + /// x: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forward(self: Self, x: Tensor, token_index: Tensor, kv_cache: KvCache, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + const dtype = x.dtype(); + + var q = self.wq.forward(x); + var k = self.wk.forward(x); + var v = self.wv.forward(x); + + q = q.splitAxis(.dout, .{ .h = attn_config.n_heads, .hd = attn_config.head_dim }); + k = k.splitAxis(.dout, .{ .h = attn_config.n_kv_heads, .hd = attn_config.head_dim }); + v = v.splitAxis(.dout, .{ .h = attn_config.n_kv_heads, .hd = attn_config.head_dim }); + + // Position indices for RoPE: token_index + arange(s) + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()) + .withTags(.{.s}).broad(Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + const rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = attn_config.rope_theta, + }; + + q = zml.nn.rope(q, pos_index, rope_opts); + k = zml.nn.rope(k, pos_index, rope_opts); + + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const cache_size = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k))), token_index.dtype()); + + if (circular_buffer) { + const cache_pos = pos_index.remainder(cache_size.broad(pos_index.shape())); + const new_kv_cache = kv_cache.update(k, v, cache_pos.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + // Reorder K/V from circular buffer to temporal order for correct causal masking. + // When the cache wraps, physical order != temporal order, which breaks FA's + // position-based causal mask. + const pos_end = token_index.addConstant(x.dim(.s)); + const is_full = pos_end.cmp(.GE, cache_size); + const rotation_start = pos_end.remainder(cache_size); + const safe_start = is_full.select(rotation_start, Tensor.scalar(@as(u32, 0), token_index.dtype())); + const reorder_arange = Tensor.arange(.{ .end = kv_cache.k.dim(.k) }, token_index.dtype()).withTags(.{.kk}); + const reorder_idx = reorder_arange.add(safe_start.broad(reorder_arange.shape())) + .remainder(cache_size.broad(reorder_arange.shape())); + k = k.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + v = v.gather(.{ .k = reorder_idx }, .{}).rename(.{ .kk = .k }); + + // Cap token_index so seqused_k doesn't exceed cache bounds + const max_token_index = Tensor.scalar(@as(u32, @intCast(kv_cache.k.dim(.k) - x.dim(.s))), token_index.dtype()); + const attn_token_index = token_index.minimum(max_token_index); + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(attn_config.sliding_window), + }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; + } else { + // Sequential: write at pos_index directly. + const new_kv_cache = kv_cache.update(k, v, pos_index.rename(.{ .s = .k })); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_token_index = token_index; + const attn_out = zml.attention.attention(q, k, v, attn_token_index, attention_metadata, attention_parameters, .{ + .sliding_window = @intCast(attn_config.sliding_window), + }); + + const merged = attn_out.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.wo.forward(merged).rename(.{ .dout = .d }), new_kv_cache }; + } + } + }; +} + +pub const SwiGluFfn = struct { + w1: zml.nn.Linear, + w2: zml.nn.Linear, + w3: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) SwiGluFfn { + return .{ + .w1 = linear(store.withPrefix("w1")), + .w2 = linear(store.withPrefix("w2")), + .w3 = linear(store.withPrefix("w3")), + }; + } + + pub fn unload(self: *zml.Bufferized(SwiGluFfn)) void { + deinitBufferized(self); + } + + /// x: [s, d] -> [s, d] + pub fn forward(self: SwiGluFfn, x: Tensor) Tensor { + const gate = self.w1.forward(x).silu(); + const up = self.w3.forward(x); + + return self.w2.forward(gate.mul(up).rename(.{ .dout = .d })).rename(.{ .dout = .d }); + } +}; + +/// KV cache for all layers of a transformer. +/// Stores K/V tensors with shape {layer, k=max_seq_len, h, hd}. +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: Shape) KvCache { + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initShape(kv_shape: Shape) zml.ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try .uninitialized(io, platform, self.k.shape(), .{}), + .v = try .uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + return .{ + .k = scatterCache(self.k, new_k, self.layer_index, token_index), + .v = scatterCache(self.v, new_v, self.layer_index, token_index), + .layer_index = self.layer_index, + }; + } + + fn scatterCache(cache: Tensor, new: Tensor, layer_index: Tensor, token_index: ?Tensor) Tensor { + const k_shape = cache.shape().drop(.layer); + const converted = new.convert(cache.dtype()).transpose(k_shape); + const scatter_opts: Tensor.ScatterOpts = .{ .indices_are_sorted = true, .update_fn = Tensor.ScatterOpts.override }; + + return if (token_index) |idx| + cache.scatterSlices(.{ .layer = layer_index.broad(idx.shape()), .k = idx }, converted, scatter_opts).reuseBuffer(cache) + else + cache.scatterSlices(.{ .layer = layer_index }, converted, scatter_opts).reuseBuffer(cache); + } + + // Unused but works for shifted kv cache + pub fn shiftIfNeeded(self: KvCache, token_index: Tensor, seq_len: u32) struct { KvCache, Tensor } { + const cache_k_size = self.k.dim(.k); + const max_pos = Tensor.scalar(@as(u32, @intCast(cache_k_size - seq_len)), token_index.dtype()); + const would_overflow = token_index.cmp(.GT, max_pos); + + const shift_arange = Tensor.arange(.{ .end = cache_k_size }, token_index.dtype()).withTags(.{.kk}); + const shifted_indices = shift_arange.addConstant(seq_len); + const max_idx = Tensor.scalar(@as(u32, @intCast(cache_k_size - 1)), token_index.dtype()); + const clamped_indices = shifted_indices.minimum(max_idx.broad(shifted_indices.shape())); + + const shifted_k = self.k.gather(.{ .k = clamped_indices }, .{}).rename(.{ .kk = .k }); + const shifted_v = self.v.gather(.{ .k = clamped_indices }, .{}).rename(.{ .kk = .k }); + + const overflow_k = would_overflow.broad(shifted_k.shape()); + const new_k = overflow_k.select(shifted_k, self.k); + const overflow_v = would_overflow.broad(shifted_v.shape()); + const new_v = overflow_v.select(shifted_v, self.v); + + return .{ .{ + .k = new_k.reuseBuffer(self.k), + .v = new_v.reuseBuffer(self.v), + .layer_index = self.layer_index, + }, token_index.minimum(max_pos) }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; diff --git a/examples/voxtral/config.zig b/examples/voxtral/config.zig new file mode 100644 index 00000000..a52947dd --- /dev/null +++ b/examples/voxtral/config.zig @@ -0,0 +1,158 @@ +const std = @import("std"); + +pub const AttentionConfig = struct { + n_heads: u32, + n_kv_heads: u32, + head_dim: u32, + rope_theta: f32, + sliding_window: u32, +}; + +pub const Config = struct { + dim: u32, + n_layers: u32, + head_dim: u32, + hidden_dim: u32, + n_heads: u32, + n_kv_heads: u32, + use_biases: bool = false, + causal: bool = true, + rope_theta: f32, + norm_eps: f32, + vocab_size: u32, + sliding_window: u32, + tied_embeddings: bool = true, + ada_rms_norm_t_cond: bool = true, + ada_rms_norm_t_cond_dim: u32 = 32, + multimodal: Multimodal, + + pub const Multimodal = struct { + whisper_model_args: WhisperModelArgs, + }; + + pub const WhisperModelArgs = struct { + encoder_args: EncoderArgs, + downsample_args: DownsampleArgs, + }; + + pub const DownsampleArgs = struct { + downsample_factor: u32, + }; + + pub const EncoderArgs = struct { + audio_encoding_args: AudioEncodingArgs, + dim: u32, + n_layers: u32, + head_dim: u32, + hidden_dim: u32, + n_heads: u32, + n_kv_heads: u32, + use_biases: bool = true, + rope_theta: f32, + norm_eps: f32, + sliding_window: u32, + + pub fn attentionConfig(self: EncoderArgs) AttentionConfig { + return .{ + .n_heads = self.n_heads, + .n_kv_heads = self.n_kv_heads, + .head_dim = self.head_dim, + .rope_theta = self.rope_theta, + .sliding_window = self.sliding_window, + }; + } + }; + + pub const AudioEncodingArgs = struct { + sampling_rate: u32 = 16000, + frame_rate: f32 = 12.5, + num_mel_bins: u32 = 128, + hop_length: u32 = 160, + window_size: u32 = 400, + global_log_mel_max: f32 = 1.5, + }; + + // -- Shortcuts + + pub fn attentionConfig(self: Config) AttentionConfig { + return .{ + .n_heads = self.n_heads, + .n_kv_heads = self.n_kv_heads, + .head_dim = self.head_dim, + .rope_theta = self.rope_theta, + .sliding_window = self.sliding_window, + }; + } + + pub fn encoder(self: Config) EncoderArgs { + return self.multimodal.whisper_model_args.encoder_args; + } + + pub fn audio(self: Config) AudioEncodingArgs { + return self.multimodal.whisper_model_args.encoder_args.audio_encoding_args; + } + + pub fn downsample_factor(self: Config) u32 { + return self.multimodal.whisper_model_args.downsample_args.downsample_factor; + } +}; + +pub fn parseConfig(allocator: std.mem.Allocator, io: std.Io, model_dir: std.Io.Dir) !std.json.Parsed(Config) { + const config_file = try model_dir.openFile(io, "params.json", .{}); // it is named params not config here + defer config_file.close(io); + + var buffer: [4096]u8 = undefined; + var file_reader = config_file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + + return try std.json.parseFromTokenSource(Config, allocator, &reader, .{ .ignore_unknown_fields = true }); +} + +pub const StreamParams = struct { + dsf: u32, + mel_per_step: u32, + chunk_audio: u32, + new_audio_per_step: u32, // mel_per_step * hop_length: new samples consumed per streaming step + audio_overlap: u32, // chunk_audio - new_audio_per_step: STFT tail reused between steps + prefill_audio_len: u32, // (prompt_len-1)*new_audio_per_step + chunk_audio: total prefill samples + raw_audio_length_per_tok: u32, + hop_length: u32, + n_delay_tokens: u32, + left_pad: u32, + prompt_len: u32, + enc_dim: u32, + + pub fn init(config: Config, transcription_delay_ms: f32, n_left_pad_tokens: u32) StreamParams { + const sample_rate: f32 = @floatFromInt(config.audio().sampling_rate); + const frame_rate = config.audio().frame_rate; + const raw_audio_length_per_tok: u32 = @intFromFloat(sample_rate / frame_rate); + const hop_length = config.audio().hop_length; + + const delay_samples: u32 = @intFromFloat(transcription_delay_ms / 1000.0 * sample_rate); + const audio_length_per_tok = raw_audio_length_per_tok / hop_length; + const n_delay_tokens = std.math.divCeil(u32, delay_samples / hop_length, audio_length_per_tok) catch unreachable; + + const dsf = config.downsample_factor(); + const mel_per_step = dsf * 2; + const window_size = config.audio().window_size; + const chunk_audio = (mel_per_step - 1) * hop_length + window_size; + const new_audio_per_step = mel_per_step * hop_length; + const prompt_len = 1 + n_left_pad_tokens + n_delay_tokens; + + return .{ + .dsf = dsf, + .mel_per_step = mel_per_step, + .chunk_audio = chunk_audio, + .new_audio_per_step = new_audio_per_step, + .audio_overlap = chunk_audio - new_audio_per_step, + .prefill_audio_len = (prompt_len - 1) * new_audio_per_step + chunk_audio, + .raw_audio_length_per_tok = raw_audio_length_per_tok, + .hop_length = hop_length, + .n_delay_tokens = n_delay_tokens, + .left_pad = n_left_pad_tokens * raw_audio_length_per_tok, + .prompt_len = prompt_len, + .enc_dim = config.encoder().dim, + }; + } +}; diff --git a/examples/voxtral/decoder.zig b/examples/voxtral/decoder.zig new file mode 100644 index 00000000..c72d76b2 --- /dev/null +++ b/examples/voxtral/decoder.zig @@ -0,0 +1,202 @@ +const std = @import("std"); + +const zml = @import("zml"); +const Tensor = zml.Tensor; + +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const common = @import("common.zig"); +const rmsNorm = common.rmsNorm; +const SwiGluFfn = common.SwiGluFfn; + +/// Adapter: encoder→decoder projection. +/// Reshapes encoder output [s, d] → [s/dsf, d*dsf] (concatenate `downsample_factor` consecutive timesteps), +/// then projects through a 2-layer MLP: Linear(d*dsf→dim) → GELU → Linear(dim→dim). +pub const Adapter = struct { + proj0: zml.nn.Linear, + proj1: zml.nn.Linear, + downsample_factor: u32, + + pub fn init(store: zml.io.TensorStore.View, config: Config) Adapter { + const proj_store = store.withPrefix("mm_streams_embeddings.embedding_module.audio_language_projection"); + + return .{ + .proj0 = common.linear(proj_store.withLayer(0)), + .proj1 = common.linear(proj_store.withLayer(2)), + .downsample_factor = config.downsample_factor(), + }; + } + + pub fn load(self: *const Adapter, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Adapter) { + return common.loadModel(Adapter, self, allocator, io, platform, store, progress, "adapter"); + } + + pub fn unload(self: *zml.Bufferized(Adapter)) void { + common.deinitBufferized(self); + } + + /// Prefill: encoder_out [s=prompt_len*dsf, d=enc_dim] → [s=prompt_len, d=dim] + /// Step: encoder_out [s=dsf, d=enc_dim] → [s=1, d=dim] + pub fn forward(self: Adapter, encoder_out: Tensor) Tensor { + var h = encoder_out; + const dsf = self.downsample_factor; + + // Reshape [s, d] → [s/dsf, d*dsf] by concatenating consecutive timesteps. + h = h.splitAxis(.s, .{ .s = .auto, .group = dsf }) + .merge(.{ .d = .{ .group, .d } }); + + return self.proj1.forward(self.proj0.forward(h).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); + } +}; + +/// Top-level decoder: runs all transformer layers, returns hidden states + updated KV cache. +pub const Decoder = struct { + /// Token embeddings: used for both input embedding and output logit computation. + tok_embeddings: Tensor, + layers: []DecoderLayer, + norm: Tensor, + norm_eps: f32, + config: Config, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Decoder { + const layers = allocator.alloc(DecoderLayer, config.n_layers) catch unreachable; + + for (layers, 0..) |*layer, i| { + layer.* = DecoderLayer.init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .tok_embeddings = store.withPrefix("mm_streams_embeddings.embedding_module.tok_embeddings").createTensorWithTags("weight", .{ .voc, .d }), + .layers = layers, + .norm = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), + .norm_eps = config.norm_eps, + .config = config, + }; + } + + pub fn deinit(self: Decoder, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn load(self: *const Decoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Decoder) { + return common.loadModel(Decoder, self, allocator, io, platform, store, progress, "decoder"); + } + + pub fn unload(self: *zml.Bufferized(Decoder), allocator: std.mem.Allocator) void { + common.deinitBufferized(self); + allocator.free(self.layers); + } + + /// Runs all decoder layers on input embeddings, samples next tokens and returns updated KV cache + RNG + incremented token index. + /// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng, next_token_index) + /// Step: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (tokens [s=1] u32, KvCache, Rng, next_token_index) + pub fn forwardStep(self: Decoder, text_tokens: Tensor, audio_embed: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, rng: Tensor.Rng, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor.Rng, Tensor } { + const text_embeds = self.tok_embeddings.gather(.{ .voc = text_tokens }, .{}); + const input_embeds = text_embeds.add(audio_embed); + + const t = t_cond.convert(input_embeds.dtype()); + + var h = input_embeds; + var cache = kv_cache; + const attn_config = self.config.attentionConfig(); + + for (self.layers, 0..) |layer, i| { + const layer_cache = cache.atLayer(i); + const result = layer.forward(h, token_index, layer_cache, t, attn_config, attention_metadata, attention_parameters); + h = result[0]; + const updated_layer_cache = result[1]; + + cache = updated_layer_cache.reuseBuffer(layer_cache); + } + + h = rmsNorm(h, self.norm, self.norm_eps); + + // Compute logits in f32 for precision + const logits = h.dot(self.tok_embeddings, .d).convert(.f32); + const output_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ .temperature = 0 }, rng); + const next_index = token_index.addConstant(1).reuseBuffer(token_index); + + return .{ output_tokens.convert(.u32), cache, new_rng, next_index }; + } +}; + +/// AdaRmsNorm: per-layer time conditioning. +/// Computes ada_scale = up(gelu(down(t_cond))), then returns h * (1 + ada_scale). +pub const AdaRmsNorm = struct { + down: zml.nn.Linear, + up: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) AdaRmsNorm { + const ada_store = store.withPrefix("ada_rms_norm_t_cond"); + return .{ + .down = common.linear(ada_store.withLayer(0)), + .up = common.linear(ada_store.withLayer(2)), + }; + } + + pub fn unload(self: *zml.Bufferized(AdaRmsNorm)) void { + common.deinitBufferized(self); + } + + /// h: [s, d], t_cond: [d] → [s, d] + pub fn forward(self: AdaRmsNorm, h: Tensor, t_cond: Tensor) Tensor { + const ada_scale = self.up.forward(self.down.forward(t_cond).gelu().rename(.{ .dout = .d })).rename(.{ .dout = .d }); + const one_plus_scale = ada_scale.addConstant(1.0).broad(h.shape()); + + return h.mul(one_plus_scale); + } +}; + +pub const KvCache = common.KvCache; + +pub const SelfAttention = common.SelfAttention(true); + +/// Single decoder transformer layer with AdaRmsNorm time conditioning. +pub const DecoderLayer = struct { + attention_norm: Tensor, + attention: SelfAttention, + ffn_norm: Tensor, + feed_forward: SwiGluFfn, + ada_norm: AdaRmsNorm, + norm_eps: f32, + + pub fn init(store: zml.io.TensorStore.View, config: Config) DecoderLayer { + return .{ + .attention_norm = store.withPrefix("attention_norm").createTensorWithTags("weight", .{.d}), + .attention = SelfAttention.init(store.withPrefix("attention")), + .ffn_norm = store.withPrefix("ffn_norm").createTensorWithTags("weight", .{.d}), + .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), + .ada_norm = AdaRmsNorm.init(store), + .norm_eps = config.norm_eps, + }; + } + + pub fn unload(self: *zml.Bufferized(DecoderLayer)) void { + common.deinitBufferized(self); + } + + /// h: [s, d], token_index: scalar, kv_cache: KvCache, t_cond: [d] → ([s, d], KvCache) + pub fn forward(self: DecoderLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, t_cond: Tensor, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + // Pre-attention norm → attention (with KV cache) → residual + const attn_out, const updated_kv_cache = self.attention.forward( + rmsNorm(h, self.attention_norm, self.norm_eps), + token_index, + kv_cache, + attn_config, + attention_metadata, + attention_parameters, + ); + + var out = h.add(attn_out); + + // Pre-FFN norm → ada conditioning → FFN → residual + const normed = rmsNorm(out, self.ffn_norm, self.norm_eps); + const conditioned = self.ada_norm.forward(normed, t_cond); + const ffn_out = self.feed_forward.forward(conditioned); + + out = out.add(ffn_out); + + return .{ out, updated_kv_cache }; + } +}; diff --git a/examples/voxtral/encoder.zig b/examples/voxtral/encoder.zig new file mode 100644 index 00000000..e1b0091d --- /dev/null +++ b/examples/voxtral/encoder.zig @@ -0,0 +1,209 @@ +const std = @import("std"); + +const zml = @import("zml"); +const Tensor = zml.Tensor; + +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const common = @import("common.zig"); +const rmsNorm = common.rmsNorm; +const KvCache = common.KvCache; + +pub const Encoder = struct { + conv0_weight: Tensor, + conv0_bias: Tensor, + conv1_weight: Tensor, + conv1_bias: Tensor, + layers: []TransformerLayer, + norm: Tensor, + norm_eps: f32, + config: Config, + + pub const ConvState = struct { + conv1: Tensor, // [batch=1, channels=128, time=2] + conv2: Tensor, // [batch=1, channels=enc_dim, time=2] + + pub fn initBuffer(self: ConvState, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvState) { + return .{ + .conv1 = try .uninitialized(io, platform, self.conv1.shape(), .{}), + .conv2 = try .uninitialized(io, platform, self.conv2.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(ConvState)) void { + self.conv1.deinit(); + self.conv2.deinit(); + } + }; + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) Encoder { + const encoder_store = store.withPrefix("mm_streams_embeddings.embedding_module.whisper_encoder"); + const conv_store = encoder_store.withPrefix("conv_layers"); + const transformer_store = encoder_store.withPrefix("transformer"); + + const enc = config.encoder(); + const layers = allocator.alloc(TransformerLayer, enc.n_layers) catch unreachable; + for (layers, 0..) |*layer, i| { + layer.* = TransformerLayer.init(transformer_store.withPrefix("layers").withLayer(i), config); + } + + const conv0_store = conv_store.withLayer(0).withPrefix("conv"); + const conv1_store = conv_store.withLayer(1).withPrefix("conv"); + + return .{ + .conv0_weight = conv0_store.createTensorWithTags("weight", .{ .cout, .cin, .k }), + .conv0_bias = conv0_store.createTensorWithTags("bias", .{.channels}), + .conv1_weight = conv1_store.createTensorWithTags("weight", .{ .cout, .cin, .k }), + .conv1_bias = conv1_store.createTensorWithTags("bias", .{.channels}), + .layers = layers, + .norm = transformer_store.withPrefix("norm").createTensorWithTags("weight", .{.d}), + .norm_eps = enc.norm_eps, + .config = config, + }; + } + + pub fn deinit(self: Encoder, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn load(self: *const Encoder, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node) !zml.Bufferized(Encoder) { + return common.loadModel(Encoder, self, allocator, io, platform, store, progress, "encoder"); + } + + pub fn unload(self: *zml.Bufferized(Encoder), allocator: std.mem.Allocator) void { + common.deinitBufferized(self); + allocator.free(self.layers); + } + + /// Prefill conv stem: runs conv on all prefill mel frames and extracts + /// conv states (last kernel_size-1=2 frames of each intermediate) for subsequent streaming. + /// Uses padding={2,0} to match reference zero-initialized state (kernel_size-1=2). + /// Prefill: mel [channels=128, time=prompt_len*mel_per_step=prompt_len*(dsf*2)] → ([s=prompt_len*dsf, d=enc_dim] bf16, ConvState) + pub fn convStemPrefill(self: Encoder, mel: Tensor) struct { Tensor, ConvState } { + const dtype = self.conv0_weight.dtype(); + const h = mel.convert(dtype).insertAxes(.channels, .{.batch}); + + // Conv0 (stride=1): pad kernel_size-1=2 zeros on left (matches zero-initialized state) + var conv0_out = h.conv1d(self.conv0_weight, .{ .window_strides = 1, .padding = &.{ 2, 0 } }); + conv0_out = conv0_out.add(self.conv0_bias.broad(conv0_out.shape())).gelu(); + + // Conv1 (stride=2): pad kernel_size-1=2 zeros on left (matches zero-initialized state) + var conv1_out = conv0_out.conv1d(self.conv1_weight, .{ .window_strides = 2, .padding = &.{ 2, 0 } }); + conv1_out = conv1_out.add(self.conv1_bias.broad(conv1_out.shape())).gelu(); + + // Output: [s=(frames+1)/2, d=enc_dim] + const output = conv1_out.squeeze(.batch) + .transpose(.{ .time, .channels }) + .rename(.{ .time = .s, .channels = .d }); + + // Extract conv states from tails (last kernel_size-1=2 frames) + const h_time: u32 = @intCast(h.dim(.time)); + const conv0_time: u32 = @intCast(conv0_out.dim(.time)); + + return .{ output, .{ + .conv1 = h.slice1d(.time, .{ .start = h_time - 2 }), + .conv2 = conv0_out.slice1d(.time, .{ .start = conv0_time - 2 }), + } }; + } + + /// Streaming conv stem: processes mel_per_step mel frames with explicit conv states. + /// Concatenates state (2 frames of left context) with new input, runs conv1d (no padding), + /// then saves the tail as new state. + /// mel_chunk: [channels=128, time=mel_per_step], conv_state: ConvState + /// Returns: ([s=dsf, d=enc_dim], new ConvState) + pub fn convStemStep(self: Encoder, mel_chunk: Tensor, conv_state: ConvState) struct { Tensor, ConvState } { + const dtype = self.conv0_weight.dtype(); + const mel_with_batch = mel_chunk.convert(dtype).insertAxes(.channels, .{.batch}); + + // Conv0 (stride=1): concat state with new mel, conv1d with no padding + const conv0_input = Tensor.concatenate(&.{ conv_state.conv1, mel_with_batch }, .time); + var conv0_out = conv0_input.conv1d(self.conv0_weight, .{ .window_strides = 1 }); + conv0_out = conv0_out.add(self.conv0_bias.broad(conv0_out.shape())).gelu(); + const mel_time: u32 = @intCast(mel_with_batch.dim(.time)); + const new_conv1_state = mel_with_batch.slice1d(.time, .{ .start = mel_time - 2 }); + + // Conv1 (stride=2): concat state with conv0 output, conv1d with no padding + const conv1_input = Tensor.concatenate(&.{ conv_state.conv2, conv0_out }, .time); + var conv1_out = conv1_input.conv1d(self.conv1_weight, .{ .window_strides = 2 }); + conv1_out = conv1_out.add(self.conv1_bias.broad(conv1_out.shape())).gelu(); + const conv0_out_time: u32 = @intCast(conv0_out.dim(.time)); + const new_conv2_state = conv0_out.slice1d(.time, .{ .start = conv0_out_time - 2 }); + + // Reshape to [s=dsf, d=enc_dim] + const output = conv1_out.squeeze(.batch) + .transpose(.{ .time, .channels }) + .rename(.{ .time = .s, .channels = .d }); + + return .{ output, .{ + .conv1 = new_conv1_state, + .conv2 = new_conv2_state, + } }; + } + + /// Runs all encoder transformer layers on conv stem output with KV cache. + /// Prefill: x [s=prompt_len*dsf, d=enc_dim] → ([s=prompt_len*dsf, d=enc_dim], KvCache, next_token_index) + /// Step: x [s=dsf, d=enc_dim] → ([s=dsf, d=enc_dim], KvCache, next_token_index) + pub fn transformerStep(self: Encoder, x: Tensor, token_index: Tensor, kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache, Tensor } { + var h = x; + var cache = kv_cache; + const attn_config = self.config.encoder().attentionConfig(); + + for (self.layers, 0..) |layer, i| { + const layer_cache = cache.atLayer(i); + const result = layer.forward(h, token_index, layer_cache, attn_config, attention_metadata, attention_parameters); + h = result[0]; + const updated_layer_cache = result[1]; + cache = updated_layer_cache.reuseBuffer(layer_cache); + } + + const output = rmsNorm(h, self.norm, self.norm_eps); + const next_index = token_index.addConstant(self.config.downsample_factor()).reuseBuffer(token_index); + return .{ output, cache, next_index }; + } +}; + +pub const TransformerLayer = struct { + attention_norm: Tensor, + attention: SelfAttention, + ffn_norm: Tensor, + feed_forward: SwiGluFfn, + norm_eps: f32, + + pub fn init(store: zml.io.TensorStore.View, config: Config) TransformerLayer { + const enc = config.encoder(); + return .{ + .attention_norm = store.withPrefix("attention_norm").createTensorWithTags("weight", .{.d}), + .attention = SelfAttention.init(store.withPrefix("attention")), + .ffn_norm = store.withPrefix("ffn_norm").createTensorWithTags("weight", .{.d}), + .feed_forward = SwiGluFfn.init(store.withPrefix("feed_forward")), + .norm_eps = enc.norm_eps, + }; + } + + pub fn unload(self: *zml.Bufferized(TransformerLayer)) void { + common.deinitBufferized(self); + } + + /// h: [s, d], token_index: scalar, kv_cache: KvCache → ([s, d], KvCache) + pub fn forward(self: TransformerLayer, h: Tensor, token_index: Tensor, kv_cache: KvCache, attn_config: cfg.AttentionConfig, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters) struct { Tensor, KvCache } { + const attn_out, const updated_kv_cache = self.attention.forward( + rmsNorm(h, self.attention_norm, self.norm_eps), + token_index, + kv_cache, + attn_config, + attention_metadata, + attention_parameters, + ); + var out = h.add(attn_out); + + const ffn_out = self.feed_forward.forward(rmsNorm(out, self.ffn_norm, self.norm_eps)); + out = out.add(ffn_out); + + return .{ out, updated_kv_cache }; + } +}; + +pub const SelfAttention = common.SelfAttention(true); + +pub const SwiGluFfn = common.SwiGluFfn; diff --git a/examples/voxtral/main.zig b/examples/voxtral/main.zig new file mode 100644 index 00000000..e617e7eb --- /dev/null +++ b/examples/voxtral/main.zig @@ -0,0 +1,235 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; +const cfg = @import("config.zig"); +const Config = cfg.Config; +const StreamParams = cfg.StreamParams; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; + +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; + +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + +const dec = @import("decoder.zig"); +const Decoder = dec.Decoder; +const Adapter = dec.Adapter; + +const common = @import("common.zig"); +const KvCache = common.KvCache; + +const voxtral = @import("voxtral.zig"); + +const CliArgs = struct { + input: ?[]const u8 = null, + model: []const u8, + transcription_delay_ms: f32 = 480.0, + backend: ?zml.attention.Backend = null, + enc_kv_size: ?u32 = null, + dec_kv_size: ?u32 = null, +}; + +const n_left_pad_tokens: u32 = 32; + +pub fn main(init: std.process.Init) !void { + log.info("Start of Voxtral", .{}); + + var dbg = std.heap.DebugAllocator(.{ .thread_safe = true }).init; + defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); + + const allocator = switch (builtin.mode) { + .Debug => dbg.allocator(), + else => std.heap.smp_allocator, + }; + + const args = stdx.flags.parse(init.minimal.args, CliArgs); + const io = init.io; + + var progress = std.Progress.start(io, .{ .root_name = "Voxtral" }); + + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + + var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer model_registry.deinit(); + + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); + + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); + defer parsed_config.deinit(); + const config = parsed_config.value; + + const sp: StreamParams = .init(config, args.transcription_delay_ms, n_left_pad_tokens); + + const delay_ms: u32 = @intFromFloat(args.transcription_delay_ms); + if (delay_ms < 80 or delay_ms > 2400 or delay_ms % 80 != 0) { + std.debug.panic("transcription_delay_ms must be a multiple of 80 in range [80, 2400], got {}", .{delay_ms}); + } + + // Build prompt_tokens: [BOS] ++ [STREAMING_PAD] * (n_left_pad_tokens + n_delay_tokens) + const prompt_tokens = try allocator.alloc(u32, sp.prompt_len); + defer allocator.free(prompt_tokens); + + var platform: *zml.Platform = try .auto(allocator, io, .{ + .cuda = .{ .allocator = .{ .bfc = .{ .memory_fraction = 0.75 } } }, + }); + defer platform.deinit(allocator); + log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + + const backend = args.backend orelse b: { + const selected = zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + var melspectro_model: LogMelSpectrogram = .init(config); + + var encoder_model: Encoder = .init(allocator, model_store.view(), config); + defer encoder_model.deinit(allocator); + + const adapter: Adapter = .init(model_store.view(), config); + + var decoder_model: Decoder = .init(allocator, model_store.view(), config); + defer decoder_model.deinit(allocator); + + const enc_cfg = config.encoder(); + const enc_kv_size = args.enc_kv_size orelse enc_cfg.sliding_window; + const dec_kv_size = args.dec_kv_size orelse config.sliding_window; + + // KV cache shapes for encoder and decoder, sized to sliding_window for memory efficiency + const enc_dtype = encoder_model.norm.dtype(); + const enc_kv_cache: KvCache = .init(.init(.{ + .layer = enc_cfg.n_layers, + .k = enc_kv_size, + .h = enc_cfg.n_kv_heads, + .hd = enc_cfg.head_dim, + }, enc_dtype)); + + const dec_dtype = decoder_model.tok_embeddings.dtype(); + const dec_kv_cache: KvCache = .init(.init(.{ + .layer = decoder_model.layers.len, + .k = dec_kv_size, + .h = config.n_kv_heads, + .hd = config.head_dim, + }, dec_dtype)); + + const enc_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(enc_kv_size))); + const dec_attention_metadata: zml.attention.Metadata = .init(.fromBackend(backend, @intCast(dec_kv_size))); + const attention_parameters: zml.attention.Parameters = .init(.fromBackend(backend)); + + // Launch concurrent compilation and buffer loading + var tokenizer_future = try io.concurrent(voxtral.loadTokenizer, .{ allocator, io, model_dir, &progress }); + + var compiled_mel_step_future = try io.concurrent(voxtral.compileMelStep, .{ allocator, io, platform, melspectro_model, sp, &progress }); + var compiled_mel_prefill_future = try io.concurrent(voxtral.compileMelPrefill, .{ allocator, io, platform, melspectro_model, sp, &progress }); + + var compiled_conv_stem_prefill_future = try io.concurrent(voxtral.compileConvStemPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len * sp.mel_per_step, &progress }); + var compiled_conv_stem_step_future = try io.concurrent(voxtral.compileConvStemStep, .{ allocator, io, platform, encoder_model, sp, &progress }); + var compiled_encoder_prefill_future = try io.concurrent(voxtral.compileEncoderPrefill, .{ allocator, io, platform, encoder_model, sp.prompt_len, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); + var compiled_encoder_step_future = try io.concurrent(voxtral.compileEncoderStep, .{ allocator, io, platform, encoder_model, enc_kv_cache, enc_attention_metadata, attention_parameters, &progress }); + var compiled_adapter_future = try io.concurrent(voxtral.compileAdapter, .{ allocator, io, platform, adapter, sp.prompt_len, config, &progress }); + var compiled_adapter_step_future = try io.concurrent(voxtral.compileAdapterStep, .{ allocator, io, platform, adapter, config, &progress }); + var compiled_decoder_future = try io.concurrent(voxtral.compileDecoder, .{ allocator, io, platform, decoder_model, sp.prompt_len, dec_kv_cache, dec_attention_metadata, attention_parameters, &progress }); + + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{ &melspectro_model, io, platform }); + var encoder_buffers_future = try io.concurrent(Encoder.load, .{ &encoder_model, allocator, io, platform, &model_store, &progress }); + var adapter_buffers_future = try io.concurrent(Adapter.load, .{ &adapter, allocator, io, platform, &model_store, &progress }); + var decoder_buffers_future = try io.concurrent(Decoder.load, .{ &decoder_model, allocator, io, platform, &model_store, &progress }); + + // Await tokenizer and look up special token IDs + var tokenizer = try tokenizer_future.await(io); + defer tokenizer.deinit(); + + const token_bos = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); + const token_streaming_pad = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); + prompt_tokens[0] = token_bos; + @memset(prompt_tokens[1..], token_streaming_pad); + + var compiled_mel_step = try compiled_mel_step_future.await(io); + defer compiled_mel_step.deinit(); + + var compiled_mel_prefill = try compiled_mel_prefill_future.await(io); + defer compiled_mel_prefill.deinit(); + + var compiled_conv_stem_prefill = try compiled_conv_stem_prefill_future.await(io); + defer compiled_conv_stem_prefill.deinit(); + + var compiled_conv_stem_step = try compiled_conv_stem_step_future.await(io); + defer compiled_conv_stem_step.deinit(); + + var compiled_encoder_prefill = try compiled_encoder_prefill_future.await(io); + defer compiled_encoder_prefill.deinit(); + + var compiled_encoder_step = try compiled_encoder_step_future.await(io); + defer compiled_encoder_step.deinit(); + + var compiled_adapter = try compiled_adapter_future.await(io); + defer compiled_adapter.deinit(); + + var compiled_adapter_step = try compiled_adapter_step_future.await(io); + defer compiled_adapter_step.deinit(); + + var compiled_decoder_prefill, var compiled_decoder_decode = try compiled_decoder_future.await(io); + defer compiled_decoder_prefill.deinit(); + defer compiled_decoder_decode.deinit(); + + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + + var encoder_buffers = try encoder_buffers_future.await(io); + defer Encoder.unload(&encoder_buffers, allocator); + + var adapter_buffers = try adapter_buffers_future.await(io); + defer Adapter.unload(&adapter_buffers); + + var decoder_buffers = try decoder_buffers_future.await(io); + defer Decoder.unload(&decoder_buffers, allocator); + + var enc_attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try enc_attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&enc_attention_metadata_buffers); + + var dec_attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try dec_attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&dec_attention_metadata_buffers); + + progress.end(); + + const conv_state: enc.Encoder.ConvState = .{ + .conv1 = Tensor.init(.{ .batch = 1, .channels = config.audio().num_mel_bins, .time = 2 }, enc_dtype), + .conv2 = Tensor.init(.{ .batch = 1, .channels = enc_cfg.dim, .time = 2 }, enc_dtype), + }; + + try voxtral.runPipeline( + allocator, + io, + platform, + config, + &tokenizer, + prompt_tokens, + sp, + .{ + .mel_step = &compiled_mel_step, + .mel_prefill = &compiled_mel_prefill, + .conv_stem_prefill = &compiled_conv_stem_prefill, + .conv_stem_step = &compiled_conv_stem_step, + .encoder_prefill = &compiled_encoder_prefill, + .encoder_step = &compiled_encoder_step, + .adapter = &compiled_adapter, + .adapter_step = &compiled_adapter_step, + .decoder_prefill = &compiled_decoder_prefill, + .decoder_decode = &compiled_decoder_decode, + }, + &mel_spectrum_buffers, + &encoder_buffers, + &adapter_buffers, + &decoder_buffers, + enc_kv_cache, + dec_kv_cache, + conv_state, + &enc_attention_metadata_buffers, + &dec_attention_metadata_buffers, + ); +} diff --git a/examples/voxtral/mel_spectrogram.zig b/examples/voxtral/mel_spectrogram.zig new file mode 100644 index 00000000..866190b6 --- /dev/null +++ b/examples/voxtral/mel_spectrogram.zig @@ -0,0 +1,114 @@ +const std = @import("std"); +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const zml = @import("zml"); +const Tensor = zml.Tensor; + +pub const LogMelSpectrogram = struct { + window: AudioWindow, + + n_fft: u63, // Must match window size + mel_filters: Tensor, // Shape must match (201, 128) + mel_floor: f32 = 1e-10, + + hop_len: u63, + global_log_mel_max: f32, + + precision: zml.DataType = .f32, + + pub fn init(config: Config) LogMelSpectrogram { + const audio = config.audio(); + + return .{ + .window = .hann, + .mel_filters = Tensor.init(.{ 201, 128 }, .f32).withTags(.{ .freq_bins, .mel }), + .hop_len = audio.hop_length, + .n_fft = audio.window_size, + .global_log_mel_max = audio.global_log_mel_max, + }; + } + + /// Computes the log mel spectrogram for a fixed-size audio chunk. + /// Prefill: audio_chunk [samples=(prompt_len-1)*mel_per_step*hop_length + chunk_audio] → [channels=128, time=prompt_len*mel_per_step] f32 + /// Step: audio_chunk [samples=chunk_audio] → [channels=128, time=mel_per_step] f32 + pub fn melStep(self: LogMelSpectrogram, audio_chunk: Tensor) Tensor { + const dtype = audio_chunk.dtype(); + + const window_weight = self.window.getWeights(self.n_fft, dtype); + const fft_len = window_weight.dim(.samples); + const audio_len: u63 = @intCast(audio_chunk.dim(.samples)); + const num_frames: u63 = @intCast(@divFloor(audio_len - fft_len, self.hop_len) + 1); + + // No reflect padding — done on host + const spectrogram = stft(audio_chunk, window_weight, num_frames, self.hop_len, self.precision); + + return self.postProcess(spectrogram, dtype).withTags(.{ .channels, .time }); + } + + /// Shared post-STFT processing: mel filter dot product, log scale, clamp, normalize. + fn postProcess(self: LogMelSpectrogram, raw_spectrogram: Tensor, dtype: zml.DataType) Tensor { + var spectrogram = raw_spectrogram.convert(dtype); + + // Re-weight frequencies for speech + spectrogram = spectrogram.dot(self.mel_filters, .freq_bins); + + spectrogram = spectrogram.maximum(Tensor.constant(dtype.constant(self.mel_floor))); + var log_spec = spectrogram.log().scale(1.0 / @log(10.0)); + + const log_spec_min = Tensor.constant(dtype.constant(self.global_log_mel_max - 8.0)); + log_spec = log_spec.maximum(log_spec_min); + + // "center" the distribution + return log_spec.addConstant(4).scale(1.0 / 4.0).transpose(.{ .mel, .frames }); + } + + pub fn load(self: *LogMelSpectrogram, io: std.Io, platform: *zml.Platform) !zml.Bufferized(LogMelSpectrogram) { + const mel_filters_data = @embedFile("assets/voxtral_mel_filter.data"); + const slice = zml.Slice.init(self.mel_filters.shape(), mel_filters_data); + + return .{ + .mel_filters = try zml.Buffer.fromSlice(io, platform, slice), + }; + } + + pub fn unload(self: *zml.Bufferized(LogMelSpectrogram)) void { + self.mel_filters.deinit(); + } +}; + +pub fn stft(waveform: Tensor, weight: Tensor, num_frames: usize, stride: u63, precision: zml.DataType) Tensor { + const fft_len = weight.dim(0); + + const indices = Tensor.arange(.{ .end = @intCast(num_frames * stride), .step = stride }, .i32); + var windows = waveform.gatherSlices(.{fft_len}, indices.appendAxes(.{.coord}), .{ .indices_are_sorted = true }); + + windows = windows.mul(weight.broadcastLeft(windows.shape())); + + var fft = windows.convert(precision).fft(.{ .kind = .RFFT, .length = &.{fft_len} }); + const spectrogram = fft.abs(); + + return spectrogram.mul(spectrogram).convert(waveform.dtype()).withTags(.{ .frames, .freq_bins }); +} + +pub const AudioWindow = enum { + /// https://numpy.org/doc/stable/reference/generated/numpy.hanning.html + hann, + boxcar, + // Other possible windows: https://en.wikipedia.org/wiki/Window_function + + pub fn getWeights(self: AudioWindow, len: i64, dtype: zml.DataType) Tensor { + return switch (self) { + .boxcar => Tensor.constant(dtype.one()).withTags(.{.samples}), + .hann => { + if (len <= 1) return Tensor.constant(dtype.one()); + + const flen: f64 = @floatFromInt(len); + const freq = Tensor.constant(dtype.constant(std.math.pi / flen)); + const steps = Tensor.arange(.{ .start = -len, .end = len, .step = 2 }, dtype); + + return steps.mul(freq).cos().scale(0.5).addConstant(0.5).convert(dtype).withTags(.{.samples}); + }, + }; + } +}; diff --git a/examples/voxtral/terminalwave/terminalwave.zig b/examples/voxtral/terminalwave/terminalwave.zig new file mode 100644 index 00000000..f1fd68ef --- /dev/null +++ b/examples/voxtral/terminalwave/terminalwave.zig @@ -0,0 +1,269 @@ +const std = @import("std"); + +// ── ANSI escape sequences & Unicode block characters ──────── + +pub const esc = struct { + // Cursor control + pub const hide_cursor = "\x1b[?25l"; + pub const show_cursor = "\x1b[?25h"; + pub const save_cursor = "\x1b7"; + pub const restore_cursor = "\x1b8"; + pub const cursor_home = "\x1b[H"; + + // Screen control + pub const clear_screen = "\x1b[2J"; + pub const clear_to_end = "\x1b[J"; + pub const reset_scroll_region = "\x1b[r"; + + // Text style + pub const reset = "\x1b[0m"; + pub const bold = "\x1b[1m"; + pub const dim_gray = "\x1b[0;90m"; + pub const bright_white = "\x1b[97m"; + + // Unicode box drawing / block elements + pub const horizontal_line = "\xe2\x94\x80"; // ─ + pub const upper_half_block = "\xe2\x96\x80"; // ▀ + pub const lower_half_block = "\xe2\x96\x84"; // ▄ + + // Parameterized format strings (for use with fmt) + pub const fg_rgb_fmt = "\x1b[38;2;{d};{d};{d}m"; + pub const bold_fg_rgb_fmt = "\x1b[1;38;2;{d};{d};{d}m"; + pub const fg_bg_rgb_fmt = "\x1b[38;2;{d};{d};{d};48;2;{d};{d};{d}m"; + pub const set_scroll_region_fmt = "\x1b[{d};999r"; + pub const move_cursor_fmt = "\x1b[{d};{d}H"; +}; + +// ── Public types ───────────────────────────────────────────── + +pub const Color = struct { r: u8, g: u8, b: u8 }; + +pub const Gradient = struct { + start: Color = .{ .r = 140, .g = 20, .b = 8 }, + mid: Color = .{ .r = 230, .g = 120, .b = 5 }, + end: Color = .{ .r = 255, .g = 220, .b = 55 }, +}; + +pub const Config = struct { + num_bars: u16 = 60, + half_height: u16 = 22, + sensitivity: f32 = 12.0, + attack: f32 = 0.6, + decay: f32 = 0.12, + gradient: Gradient = .{}, + padding_left: u16 = 4, + title: []const u8, + show_title: bool = true, +}; + +pub const FrameResult = struct { + level: f32, +}; + +// ── State ──────────────────────────────────────────────────── + +pub const State = struct { + config: Config, + writer: *std.Io.Writer, + smoothed: f32, + frame_count: u32, + buf: [BUF_SIZE]u8, + pos: usize, + + gradient_lut: [MAX_HALF_HEIGHT]Color, + + const BUF_SIZE = 131072; + const MAX_HALF_HEIGHT = 256; + const Half = enum { upper, lower }; + + pub fn init(config: Config, writer: *std.Io.Writer) State { + var state = State{ + .config = config, + .writer = writer, + .smoothed = 0, + .frame_count = 0, + .gradient_lut = undefined, + .buf = undefined, + .pos = 0, + }; + + state.buildGradientLut(); + + writer.writeAll(esc.hide_cursor ++ esc.clear_screen) catch {}; + + return state; + } + + pub fn deinit(self: *State) void { + self.writer.writeAll(esc.reset_scroll_region ++ esc.show_cursor) catch {}; + } + + pub fn render(self: *State, rms: f32) FrameResult { + // Smoothing + const attack_rate: f32 = if (rms > self.smoothed) self.config.attack else self.config.decay; + self.smoothed = self.smoothed * (1.0 - attack_rate) + rms * attack_rate; + const level = @min(self.smoothed * self.config.sensitivity, 1.0); + + // Render + self.pos = 0; + self.put(esc.cursor_home); + + if (self.config.show_title) { + self.put("\n"); + self.fmt(esc.bold_fg_rgb_fmt, .{ 230, 120, 5 }); + self.putPadding(); + self.put(self.config.title); + self.put(" " ++ esc.dim_gray ++ esc.horizontal_line ++ " live transcription" ++ esc.reset ++ "\n\n"); + } + + const num_bars = self.config.num_bars; + const half_height = self.config.half_height; + const half_term = half_height / 2; + + // Compute bar heights + var heights: [512]f32 = undefined; + const center: f32 = @as(f32, @floatFromInt(num_bars)) / 2.0; + const t: f32 = @as(f32, @floatFromInt(self.frame_count)) * 0.12; + + for (0..num_bars) |i| { + const fi: f32 = @as(f32, @floatFromInt(i)); + const dist = (fi - center) / center; + const envelope = @exp(-dist * dist * 3.5); + + const w1 = 0.28 * @sin(fi * 0.8 + t); + const w2 = 0.18 * @sin(fi * 1.5 - t * 1.6); + const w3 = 0.12 * @sin(fi * 2.4 + t * 0.7); + const w4 = 0.08 * @sin(fi * 3.7 - t * 2.1); + const wobble = w1 + w2 + w3 + w4; + + const h = level * envelope * (1.0 + wobble) * @as(f32, @floatFromInt(half_height)); + heights[i] = @max(@min(h, @as(f32, @floatFromInt(half_height))), 0.0); + } + + self.renderHalf(.upper, half_term, half_height, num_bars, heights[0..num_bars]); + self.renderHalf(.lower, half_term, half_height, num_bars, heights[0..num_bars]); + + self.frame_count +%= 1; + + return .{ .level = level }; + } + + /// Render a horizontal separator line below the wave visualization. + pub fn renderSeparator(self: *State) void { + self.put("\n"); + self.fmt(esc.fg_rgb_fmt, .{ 60, 60, 60 }); + self.putPadding(); + for (0..self.config.num_bars) |_| { + self.put(esc.horizontal_line); + } + self.put(esc.reset ++ "\n\n"); + } + + // ── Private helpers ────────────────────────────────────── + + fn buildGradientLut(self: *State) void { + const hh = self.config.half_height; + const g = self.config.gradient; + + for (0..hh) |i| { + const t: f32 = @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(hh - 1)); + if (t < 0.5) { + const s = t * 2.0; + self.gradient_lut[i] = .{ + .r = lerpU8(@floatFromInt(g.start.r), @floatFromInt(g.mid.r), s), + .g = lerpU8(@floatFromInt(g.start.g), @floatFromInt(g.mid.g), s), + .b = lerpU8(@floatFromInt(g.start.b), @floatFromInt(g.mid.b), s), + }; + } else { + const s = (t - 0.5) * 2.0; + self.gradient_lut[i] = .{ + .r = lerpU8(@floatFromInt(g.mid.r), @floatFromInt(g.end.r), s), + .g = lerpU8(@floatFromInt(g.mid.g), @floatFromInt(g.end.g), s), + .b = lerpU8(@floatFromInt(g.mid.b), @floatFromInt(g.end.b), s), + }; + } + } + } + + fn renderHalf(self: *State, half: Half, half_term: u16, half_height: u16, num_bars: u16, heights: []const f32) void { + for (0..half_term) |tr| { + self.putPadding(); + const tr16 = @as(u16, @intCast(tr)); + const upper_log, const lower_log = switch (half) { + .upper => .{ half_height - 1 - 2 * tr16, half_height - 2 - 2 * tr16 }, + .lower => .{ 2 * tr16 + 1, 2 * tr16 + 2 }, + }; + for (0..num_bars) |col| { + const h = heights[col]; + const upper_on = h > @as(f32, @floatFromInt(upper_log)); + const lower_on = h > @as(f32, @floatFromInt(lower_log)); + + if (upper_on and lower_on) { + self.cellBoth(self.gradient_lut[upper_log], self.gradient_lut[lower_log]); + } else if (half == .upper and lower_on) { + self.cellLower(self.gradient_lut[lower_log]); + } else if (half == .lower and upper_on) { + self.cellUpper(self.gradient_lut[upper_log]); + } else { + self.put(" "); + } + } + self.put("\n"); + } + } + + // ── Public buffer helpers (for caller to append after render) ── + + pub fn putPadding(self: *State) void { + for (0..self.config.padding_left) |_| { + self.put(" "); + } + } + + pub fn put(self: *State, s: []const u8) void { + if (self.pos + s.len > BUF_SIZE) return; + @memcpy(self.buf[self.pos..][0..s.len], s); + self.pos += s.len; + } + + pub fn fmt(self: *State, comptime f: []const u8, args: anytype) void { + const result = std.fmt.bufPrint(self.buf[self.pos..], f, args) catch return; + self.pos += result.len; + } + + pub fn flush(self: *State) void { + self.writer.writeAll(self.buf[0..self.pos]) catch {}; + self.pos = 0; + } + + fn cellBoth(self: *State, upper: Color, lower: Color) void { + self.fmt(esc.fg_bg_rgb_fmt, .{ upper.r, upper.g, upper.b, lower.r, lower.g, lower.b }); + self.put(esc.upper_half_block ++ esc.reset); + } + + fn cellUpper(self: *State, col: Color) void { + self.fmt(esc.fg_rgb_fmt, .{ col.r, col.g, col.b }); + self.put(esc.upper_half_block ++ esc.reset); + } + + fn cellLower(self: *State, col: Color) void { + self.fmt(esc.fg_rgb_fmt, .{ col.r, col.g, col.b }); + self.put(esc.lower_half_block ++ esc.reset); + } +}; + +// ── Free functions ─────────────────────────────────────────── + +fn lerpU8(a: f32, b: f32, t: f32) u8 { + return @intFromFloat(@max(0.0, @min(255.0, a + (b - a) * t))); +} + +/// Compute RMS (root mean square) of audio samples for level metering. +pub fn computeRms(samples: []const f32) f32 { + if (samples.len == 0) return 0; + var sum: f32 = 0; + for (samples) |s| { + sum += s * s; + } + return @sqrt(sum / @as(f32, @floatFromInt(samples.len))); +} diff --git a/examples/voxtral/test.zig b/examples/voxtral/test.zig new file mode 100644 index 00000000..d268cbc3 --- /dev/null +++ b/examples/voxtral/test.zig @@ -0,0 +1,240 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const log = std.log; +const cfg = @import("config.zig"); +const Config = cfg.Config; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; + +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; + +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + +const dec = @import("decoder.zig"); + +const CliArgs = struct { + reference: []const u8, + model: []const u8, +}; + +pub fn main(init: std.process.Init) !void { + log.info("Start of Voxtral test", .{}); + + var dbg = std.heap.DebugAllocator(.{ .thread_safe = true }).init; + defer if (builtin.mode == .Debug) std.debug.assert(dbg.deinit() == .ok); + + const allocator = switch (builtin.mode) { + .Debug => dbg.allocator(), + else => std.heap.c_allocator, + }; + + const args = stdx.flags.parse(init.minimal.args, CliArgs); + + const io = init.io; + + var ref_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.reference); + defer ref_registry.deinit(); + + var ref_store: zml.io.TensorStore = .fromRegistry(allocator, &ref_registry); + defer ref_store.deinit(); + + var model_registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer model_registry.deinit(); + + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); + + // Parse config from model directory + const model_dir = try zml.safetensors.resolveModelRepo(io, args.model); + var parsed_config = try cfg.parseConfig(allocator, io, model_dir); + defer parsed_config.deinit(); + const config = parsed_config.value; + + const encoder_prefix = "mm_streams_embeddings.embedding_module.whisper_encoder"; + const load_opts: zml.io.LoadOpts = .{ + .store = &model_store, + .parallelism = 16, + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + }; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("Selected platform {f}\n", .{platform.fmtVerbose()}); + + // -- Mel spectrogram test + { + var melspectro_model: LogMelSpectrogram = .init(config); + var mel_spectrum_buffers_future = try io.concurrent(LogMelSpectrogram.load, .{&melspectro_model, io, platform}); + var mel_spectrum_buffers = try mel_spectrum_buffers_future.await(io); + defer LogMelSpectrogram.unload(&mel_spectrum_buffers); + + try zml.testing.testLayer(allocator, io, platform, MelTestHarness{ .inner = melspectro_model }, .forward, ref_store.view(), "mel", .{ .inner = mel_spectrum_buffers }, .{}); + } + + // -- Conv stem test + { + const conv_store = model_store.view().withPrefix(encoder_prefix).withPrefix("conv_layers"); + const conv_stem = ConvStemTestHarness{ + .conv0 = enc.CausalConv1d.init(conv_store.withLayer(0).withPrefix("conv"), 1), + .conv1 = enc.CausalConv1d.init(conv_store.withLayer(1).withPrefix("conv"), 2), + }; + var conv_buffers = try zml.io.load(ConvStemTestHarness, &conv_stem, allocator, io, platform, load_opts); + defer ConvStemTestHarness.unload(&conv_buffers); + + try zml.testing.testLayer(allocator, io, platform, conv_stem, .forward, ref_store.view(), "conv_stem", conv_buffers, .{}); + } + + // -- Adapter test + { + const adapter = AdapterTestHarness{ + .inner = dec.Adapter.init(model_store.view(), config), + }; + var adapter_buffers = try zml.io.load(AdapterTestHarness, &adapter, allocator, io, platform, load_opts); + defer AdapterTestHarness.unload(&adapter_buffers); + + try zml.testing.testLayer(allocator, io, platform, adapter, .forward, ref_store.view(), "adapter", adapter_buffers, .{.minimum_close_fraction = 0.8}); + } + + // -- Decoder Layer tests + { + const dec_layer_store = model_store.view().withPrefix("layers"); + + // Load t_cond from reference activations (shared across all layers) + const t_cond_tensor = ref_store.view().withPrefix("t_cond.out").createTensorWithTags("0", .{.d}); + const TCond = struct { t_cond: Tensor }; + const t_cond_spec = TCond{ .t_cond = t_cond_tensor }; + const ref_load_opts: zml.io.LoadOpts = .{ + .store = &ref_store, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 4096, + }; + var t_cond_buf = try zml.io.load(TCond, &t_cond_spec, allocator, io, platform, ref_load_opts); + defer t_cond_buf.t_cond.deinit(); + + inline for (0..32) |i| { + if (i < config.n_layers) { + const layer = DecoderLayerTestHarness{ + .inner = dec.DecoderLayer.init(dec_layer_store.withLayer(i), config), + .t_cond = t_cond_tensor, + .config = config, + }; + + var inner_buffers = try zml.io.load(dec.DecoderLayer, &layer.inner, allocator, io, platform, load_opts); + defer dec.DecoderLayer.unload(&inner_buffers); + + const layer_buffers: zml.Bufferized(DecoderLayerTestHarness) = .{ + .inner = inner_buffers, + .t_cond = t_cond_buf.t_cond, + }; + + try zml.testing.testLayer(allocator, io, platform, layer, .forward, ref_store.view(), std.fmt.comptimePrint("decoder.prefill.layers.{d}", .{i}), layer_buffers, .{ .minimum_close_fraction = 0.85 }); + } + } + } +} + +// ================================================================ +// Test harnesses +// ================================================================ + +const MelTestHarness = struct { + inner: LogMelSpectrogram, + + pub fn forward(self: MelTestHarness, waveform: Tensor) Tensor { + return self.inner.forward(waveform.withTags(.{.samples})); + } +}; + +const ConvStemTestHarness = struct { + conv0: enc.CausalConv1d, + conv1: enc.CausalConv1d, + + pub fn unload(self: *zml.Bufferized(ConvStemTestHarness)) void { + enc.CausalConv1d.unload(&self.conv0); + enc.CausalConv1d.unload(&self.conv1); + } + + pub fn forward(self: ConvStemTestHarness, mel_input: Tensor) Tensor { + const dtype = self.conv0.weight.dtype(); + var h = mel_input.convert(dtype).withTags(.{ .channels, .time }).insertAxes(.channels, .{.batch}); + + h = self.conv0.forward(h).gelu(); + h = self.conv1.forward(h).gelu(); + h = h.squeeze(.batch); + + return h.transpose(.{ .time, .channels }).convert(.f32); + } +}; + +const TransformerLayerTestHarness = struct { + inner: enc.TransformerLayer, + config: Config, + + pub fn unload(self: *zml.Bufferized(TransformerLayerTestHarness)) void { + enc.TransformerLayer.unload(&self.inner); + } + + pub fn forward(self: TransformerLayerTestHarness, h: Tensor) Tensor { + const dtype = self.inner.attention.wq.weight.dtype(); + return self.inner.forward(h.convert(dtype).withTags(.{ .s, .d }), self.config).convert(.f32); + } +}; + +const EncoderTestHarness = struct { + inner: Encoder, + + pub fn forward(self: EncoderTestHarness, mel_input: Tensor) Tensor { + return self.inner.forward(mel_input.withTags(.{ .channels, .time })).convert(.f32); + } +}; + +const AdapterTestHarness = struct { + inner: dec.Adapter, + + pub fn unload(self: *zml.Bufferized(AdapterTestHarness)) void { + dec.Adapter.unload(&self.inner); + } + + pub fn forward(self: AdapterTestHarness, encoder_out: Tensor) Tensor { + return self.inner.forward(encoder_out.convert(.bf16).withTags(.{ .s, .d })).convert(.f32); + } +}; + +const DecoderLayerTestHarness = struct { + inner: dec.DecoderLayer, + t_cond: Tensor, + config: Config, + + pub fn forward(self: DecoderLayerTestHarness, h: Tensor) Tensor { + const dtype = self.inner.attention.wq.weight.dtype(); + const h_typed = h.convert(dtype).withTags(.{ .s, .d }); + + // Create zero KV cache for prefill (empty cache for this layer) + const attn_config = self.config.attentionConfig(); + const kv_shape = zml.Shape.init(.{ + .layer = 1, + .k = h_typed.dim(.s), + .h = @as(i64, @intCast(attn_config.n_kv_heads)), + .hd = @as(i64, @intCast(attn_config.head_dim)), + }, dtype); + + const kv_cache = dec.KvCache{ + .k = Tensor.zeroes(kv_shape), + .v = Tensor.zeroes(kv_shape), + .layer_index = Tensor.scalar(0, .u32), + }; + + const token_index = Tensor.scalar(0, .u32); + const attention_metadata: zml.attention.Metadata = .{ .vanilla = {} }; + const attention_parameters: zml.attention.Parameters = .{ .vanilla = {} }; + const result = self.inner.forward(h_typed, token_index, kv_cache, self.t_cond.convert(dtype), attn_config, attention_metadata, attention_parameters); + + return result[0].convert(.f32); + } +}; diff --git a/examples/voxtral/voxtral.zig b/examples/voxtral/voxtral.zig new file mode 100644 index 00000000..a3823670 --- /dev/null +++ b/examples/voxtral/voxtral.zig @@ -0,0 +1,673 @@ +const std = @import("std"); +const log = std.log; +const cfg = @import("config.zig"); +const Config = cfg.Config; +const StreamParams = cfg.StreamParams; + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; + +const mel = @import("mel_spectrogram.zig"); +const LogMelSpectrogram = mel.LogMelSpectrogram; + +const enc = @import("encoder.zig"); +const Encoder = enc.Encoder; + +const dec = @import("decoder.zig"); +const Decoder = dec.Decoder; +const Adapter = dec.Adapter; + +const common = @import("common.zig"); +const KvCache = common.KvCache; + +const terminalwave = @import("terminalwave/terminalwave.zig"); +const tesc = terminalwave.esc; + +pub const CompiledExes = struct { + mel_step: *zml.Exe, + mel_prefill: *zml.Exe, + conv_stem_prefill: *zml.Exe, + conv_stem_step: *zml.Exe, + encoder_prefill: *zml.Exe, + encoder_step: *zml.Exe, + adapter: *zml.Exe, + adapter_step: *zml.Exe, + decoder_prefill: *zml.Exe, + decoder_decode: *zml.Exe, +}; + +pub const ModelBuffers = struct { + mel_spectrum: *zml.Bufferized(LogMelSpectrogram), + encoder: *zml.Bufferized(Encoder), + adapter: *zml.Bufferized(Adapter), + decoder: *zml.Bufferized(Decoder), + enc_kv: *zml.Bufferized(KvCache), + dec_kv: *zml.Bufferized(KvCache), + conv_state: *zml.Bufferized(Encoder.ConvState), + t_cond: zml.Buffer, + rng: *zml.Bufferized(Tensor.Rng), + enc_attention_metadata: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata: *zml.Bufferized(zml.attention.Metadata), +}; + +pub const PipelineContext = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + sp: StreamParams, + exes: CompiledExes, + buffers: ModelBuffers, +}; + +fn compileStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: anytype, comptime method: anytype, inputs: anytype, progress: *std.Progress.Node, comptime label: []const u8) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling " ++ label ++ "...", 1); + defer node.end(); + return try platform.compile(allocator, io, model, method, inputs); +} + +/// Compiles LogMelSpectrogram.melStep for streaming (one chunk at a time). +/// Input: [samples=chunk_audio] f32 +/// Output: [channels=128, time=mel_per_step] f32 +pub fn compileMelStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + return compileStep(allocator, io, platform, model, .melStep, .{Tensor.init(.{sp.chunk_audio}, .f32).withTags(.{.samples})}, progress, "mel step"); +} + +/// Compiles LogMelSpectrogram.melStep for the full prefill audio. +/// Input: [samples=(prompt_len-1)*mel_per_step*hop_length + chunk_audio] f32 +/// Output: [channels=128, time=prompt_len*mel_per_step] f32 +pub fn compileMelPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: LogMelSpectrogram, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + const full_audio: u32 = (sp.prompt_len - 1) * sp.mel_per_step * sp.hop_length + sp.chunk_audio; + return compileStep(allocator, io, platform, model, .melStep, .{ + Tensor.init(.{full_audio}, .f32).withTags(.{.samples}), + }, progress, "mel prefill"); +} + +/// Compiles Encoder.convStemPrefill for the full prefill mel spectrogram. +/// Input: [channels=128, time=num_mel_frames] f32 (num_mel_frames = prompt_len*mel_per_step) +/// Output: ([s=num_mel_frames/2=prompt_len*dsf, d=enc_dim] bf16, ConvState) +pub fn compileConvStemPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, num_mel_frames: u32, progress: *std.Progress.Node) !zml.Exe { + return compileStep(allocator, io, platform, model, .convStemPrefill, .{ + Tensor.init(.{ .channels = 128, .time = num_mel_frames }, .f32), + }, progress, "conv stem prefill"); +} + +/// Compiles Encoder.convStemStep for one streaming step. +/// Input: [channels=128, time=mel_per_step] f32, ConvState +/// Output: ([s=dsf, d=enc_dim] bf16, ConvState) +pub fn compileConvStemStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, sp: StreamParams, progress: *std.Progress.Node) !zml.Exe { + const dtype = model.conv0_weight.dtype(); + const enc_dim = model.config.encoder().dim; + return compileStep(allocator, io, platform, model, .convStemStep, .{ + Tensor.init(.{ .channels = 128, .time = sp.mel_per_step }, .f32), + Encoder.ConvState{ + .conv1 = Tensor.init(.{ .batch = 1, .channels = 128, .time = 2 }, dtype), + .conv2 = Tensor.init(.{ .batch = 1, .channels = enc_dim, .time = 2 }, dtype), + }, + }, progress, "conv stem step"); +} + +/// Compiles Encoder.transformer for prefill: processes prompt_len*dsf frames, populating the KV cache. +/// Input: x [s=prompt_len*dsf, d=enc_dim] bf16, token_index scalar u32 +/// Output: ([s=prompt_len*dsf, d=enc_dim] bf16, KvCache) +pub fn compileEncoderPrefill(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, prompt_len: u32, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { + const enc_cfg = model.config.encoder(); + const dsf = model.config.downsample_factor(); + return compileStep(allocator, io, platform, model, .transformerStep, .{ + Tensor.init(.{ .s = prompt_len * dsf, .d = enc_cfg.dim }, .bf16), + Tensor.init(.{}, .u32), + enc_kv_cache, + attention_metadata, + attention_parameters, + }, progress, "encoder prefill"); +} + +/// Compiles Encoder.transformerStep for streaming: processes dsf frames using the KV cache. +/// Input: x [s=dsf, d=enc_dim] bf16, token_index scalar u32 +/// Output: ([s=dsf, d=enc_dim] bf16, KvCache, token_index scalar u32) +pub fn compileEncoderStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Encoder, enc_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !zml.Exe { + const enc_cfg = model.config.encoder(); + const dsf = model.config.downsample_factor(); + return compileStep(allocator, io, platform, model, .transformerStep, .{ + Tensor.init(.{ .s = dsf, .d = enc_cfg.dim }, .bf16), + Tensor.init(.{}, .u32), + enc_kv_cache, + attention_metadata, + attention_parameters, + }, progress, "encoder step"); +} + +/// Compiles Adapter.forward for prefill: maps prompt_len*dsf encoder frames to prompt_len decoder embeddings. +/// Input: [s=prompt_len*dsf, d=enc_dim] bf16 +/// Output: [s=prompt_len, d=dim] bf16 +pub fn compileAdapter(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, prompt_len: u32, config: Config, progress: *std.Progress.Node) !zml.Exe { + const dsf = config.downsample_factor(); + return compileStep(allocator, io, platform, model, .forward, .{ + Tensor.init(.{ .s = prompt_len * dsf, .d = config.encoder().dim }, .bf16), + }, progress, "adapter"); +} + +/// Compiles Adapter.forward for streaming: maps dsf encoder frames to 1 decoder embedding. +/// Input: [s=dsf, d=enc_dim] bf16 +/// Output: [s=1, d=dim] bf16 +pub fn compileAdapterStep(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Adapter, config: Config, progress: *std.Progress.Node) !zml.Exe { + const dsf = config.downsample_factor(); + return compileStep(allocator, io, platform, model, .forward, .{ + Tensor.init(.{ .s = dsf, .d = config.encoder().dim }, .bf16), + }, progress, "adapter step"); +} + +/// Compiles Decoder.forward (prefill) and Decoder.forwardStep (decode step) concurrently. +/// Prefill: text_tokens [s=prompt_len] u32, audio_embed [s=prompt_len, d=dim] → (tokens [s=prompt_len] u32, KvCache, Rng) +/// Decode: text_tokens [s=1] u32, audio_embed [s=1, d=dim] → (token [s=1] u32, KvCache, Rng, token_index scalar u32) +pub fn compileDecoder(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, model: Decoder, prompt_len: u32, dec_kv_cache: KvCache, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, progress: *std.Progress.Node) !struct { zml.Exe, zml.Exe } { + const dim = model.config.dim; + + const compilePrefill = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, s: u32, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { + return compileStep(allocator_, io_, platform_, model_, .forwardStep, .{ + Tensor.init(.{ .s = s }, .u32), + Tensor.init(.{ .s = s, .d = dim_ }, .bf16), + Tensor.init(.{}, .u32), + kv_cache_, + Tensor.init(.{ .d = dim_ }, .f32), + Tensor.Rng.init(), + attention_metadata_, + attention_parameters_, + }, progress_, "decoder prefill"); + } + }.call; + + const compileDecode = struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, model_: Decoder, kv_cache_: KvCache, dim_: u32, attention_metadata_: zml.attention.Metadata, attention_parameters_: zml.attention.Parameters, progress_: *std.Progress.Node) !zml.Exe { + return compileStep(allocator_, io_, platform_, model_, .forwardStep, .{ + Tensor.init(.{ .s = 1 }, .u32), + Tensor.init(.{ .s = 1, .d = dim_ }, .bf16), + Tensor.init(.{}, .u32), + kv_cache_, + Tensor.init(.{ .d = dim_ }, .f32), + Tensor.Rng.init(), + attention_metadata_, + attention_parameters_, + }, progress_, "decoder decode"); + } + }.call; + + var prefill_future = try io.concurrent(compilePrefill, .{ allocator, io, platform, model, prompt_len, dec_kv_cache, dim, attention_metadata, attention_parameters, progress }); + var decode_future = try io.concurrent(compileDecode, .{ allocator, io, platform, model, dec_kv_cache, dim, attention_metadata, attention_parameters, progress }); + + return .{ + try prefill_future.await(io), + try decode_future.await(io), + }; +} + +pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); +} + +/// Sinusoidal time embedding: encodes a scalar t into a [dim]-dimensional vector. +/// out[i] = cos(t * inv_freq[i]) for i < half_dim +/// out[i] = sin(t * inv_freq[i-half_dim]) for i >= half_dim +/// where inv_freq[i] = exp(-log(theta) * i / half_dim) +pub fn computeTimeEmbedding(allocator: std.mem.Allocator, t_value: f32, dim: u32) []f32 { + const half_dim = dim / 2; + const result = allocator.alloc(f32, dim) catch unreachable; + + for (0..half_dim) |i| { + const freq = @exp(-@log(@as(f32, 10000.0)) * @as(f32, @floatFromInt(i)) / @as(f32, @floatFromInt(half_dim))); + const angle = t_value * freq; + result[i] = @cos(angle); + result[i + half_dim] = @sin(angle); + } + + return result; +} + +/// Run the prefill phase: mel -> conv stem -> encoder -> adapter -> decoder prefill. +/// Populates KV caches and produces the first generated token into generated_token_slice. +fn runPrefill(ctx: *PipelineContext, padded_audio: []const f32, prompt_tokens: []const u32, generated_token_slice: zml.Slice) !void { + const allocator = ctx.allocator; + const io = ctx.io; + const platform = ctx.platform; + const config = ctx.config; + const sp = ctx.sp; + + const enc_dim = sp.enc_dim; + const prompt_len = sp.prompt_len; + + // Mel prefill: single kernel, produces [channels=128, time=prompt_len*mel_per_step] on device + const full_audio_len = sp.prefill_audio_len; + log.info("Running mel prefill ({} audio samples -> {} mel frames)...", .{ full_audio_len, sp.prompt_len * sp.mel_per_step }); + var mel_prefill_output: zml.Buffer = undefined; + { + const audio_slice: zml.Slice = .init( + .init(.{full_audio_len}, .f32), + std.mem.sliceAsBytes(padded_audio[0..full_audio_len]), + ); + var audio_buffer: zml.Buffer = try .fromSlice(io, platform, audio_slice); + defer audio_buffer.deinit(); + + mel_prefill_output = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.prompt_len * sp.mel_per_step }, .f32), .{}); + + var args = try ctx.exes.mel_prefill.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.mel_prefill.results(allocator); + defer results.deinit(allocator); + + args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); + ctx.exes.mel_prefill.call(args, &results); + results.fill(.{&mel_prefill_output}); + } + defer mel_prefill_output.deinit(); + log.info("Mel prefill done ({} frames).", .{sp.prompt_len * sp.mel_per_step}); + + // Conv stem prefill: [channels=128, time=prompt_len*mel_per_step=prompt_len*(dsf*2)] -> [s=prompt_len*dsf, d=enc_dim] + // Also extracts conv states (last 2 frames of each intermediate) for streaming + log.info("Running conv stem prefill ({} mel frames)...", .{prompt_len * sp.mel_per_step}); + var conv_prefill_output: zml.Buffer = undefined; + { + const prefill_enc_frames = prompt_len * sp.dsf; + conv_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_enc_frames, .d = enc_dim }, .bf16), .{}); + + var args = try ctx.exes.conv_stem_prefill.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.conv_stem_prefill.results(allocator); + defer results.deinit(allocator); + + args.set(.{ ctx.buffers.encoder, mel_prefill_output }); + ctx.exes.conv_stem_prefill.call(args, &results); + results.fill(.{ &conv_prefill_output, ctx.buffers.conv_state }); + } + defer conv_prefill_output.deinit(); + log.info("Conv stem prefill done.", .{}); + + // Encoder prefill: first prompt_len*dsf frames through transformer with KV cache + log.info("Running encoder prefill ({} frames)...", .{prompt_len * sp.dsf}); + var encoder_prefill_output: zml.Buffer = undefined; + { + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer enc_token_index.deinit(); + + const prefill_frames = prompt_len * sp.dsf; + encoder_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prefill_frames, .d = enc_dim }, .bf16), .{}); + + var args = try ctx.exes.encoder_prefill.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.encoder_prefill.results(allocator); + defer results.deinit(allocator); + + args.set(.{ ctx.buffers.encoder, conv_prefill_output, enc_token_index, ctx.buffers.enc_kv, ctx.buffers.enc_attention_metadata }); + ctx.exes.encoder_prefill.call(args, &results); + results.fill(.{ &encoder_prefill_output, ctx.buffers.enc_kv }); + } + defer encoder_prefill_output.deinit(); + log.info("Encoder prefill done.", .{}); + + // Adapter batch: encoder prefill output -> prompt_len audio embeddings + log.info("Running adapter...", .{}); + var adapter_prefill_output: zml.Buffer = undefined; + { + adapter_prefill_output = try .uninitialized(io, platform, .init(.{ .s = prompt_len, .d = config.dim }, .bf16), .{}); + + var args = try ctx.exes.adapter.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.adapter.results(allocator); + defer results.deinit(allocator); + + args.set(.{ ctx.buffers.adapter, encoder_prefill_output }); + ctx.exes.adapter.call(args, &results); + results.fill(.{&adapter_prefill_output}); + } + defer adapter_prefill_output.deinit(); + log.info("Adapter done.", .{}); + + // Decoder prefill: all prompt tokens + audio embeddings -> first generated token + log.info("Running decoder prefill...", .{}); + { + var args = try ctx.exes.decoder_prefill.args(allocator); + defer args.deinit(allocator); + var results = try ctx.exes.decoder_prefill.results(allocator); + defer results.deinit(allocator); + + const token_data_slice: zml.Slice = .init( + .init(.{prompt_len}, .u32), + std.mem.sliceAsBytes(prompt_tokens), + ); + var token_buffer: zml.Buffer = try .fromSlice(io, platform, token_data_slice); + defer token_buffer.deinit(); + + var token_index_buffer: zml.Buffer = try .scalar(io, platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + + var prefill_output_slice: zml.Slice = try .alloc(allocator, .init(.{prompt_len}, .u32)); + defer prefill_output_slice.free(allocator); + var prefill_output: zml.Buffer = try .uninitialized(io, platform, .init(.{prompt_len}, .u32), .{}); + defer prefill_output.deinit(); + + args.set(.{ ctx.buffers.decoder, token_buffer, adapter_prefill_output, token_index_buffer, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); + ctx.exes.decoder_prefill.call(args, &results); + results.fill(.{ &prefill_output, ctx.buffers.dec_kv, ctx.buffers.rng }); + + // Extract last token from prefill output (first generated token) + try prefill_output.toSlice(io, prefill_output_slice); + generated_token_slice.items(u32)[0] = prefill_output_slice.items(u32)[prompt_len - 1]; + } + log.info("Decoder prefill done.\n", .{}); +} + +/// Run the streaming decode loop: step-by-step conv_stem -> encoder -> adapter -> decoder. +fn runGenerationLoop(ctx: *PipelineContext, tokenizer: *zml.tokenizer.Tokenizer, initial_overlap: []const f32, stdin_reader: *std.Io.Reader, generated_token_slice: zml.Slice) !void { + const allocator = ctx.allocator; + const io = ctx.io; + const platform = ctx.platform; + const config = ctx.config; + const sp = ctx.sp; + + const enc_dim = sp.enc_dim; + const prompt_len = sp.prompt_len; + const new_audio_per_step = sp.new_audio_per_step; + const audio_overlap = sp.audio_overlap; + + // Sliding audio buffer: [overlap from previous step | new samples] + const audio_buf = try allocator.alloc(f32, sp.chunk_audio); + defer allocator.free(audio_buf); + @memcpy(audio_buf[0..audio_overlap], initial_overlap); + @memset(audio_buf[audio_overlap..], 0); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const eos_token = tokenizer.tokenToId("") orelse @panic("tokenizer missing token"); + const streaming_pad_token = tokenizer.tokenToId("[STREAMING_PAD]") orelse @panic("tokenizer missing [STREAMING_PAD] token"); + const streaming_word_token = tokenizer.tokenToId("[STREAMING_WORD]"); + + const decode_start: std.Io.Timestamp = .now(io, .awake); + + var mel_step_args = try ctx.exes.mel_step.args(allocator); + defer mel_step_args.deinit(allocator); + var mel_step_results = try ctx.exes.mel_step.results(allocator); + defer mel_step_results.deinit(allocator); + + var conv_step_args = try ctx.exes.conv_stem_step.args(allocator); + defer conv_step_args.deinit(allocator); + var conv_step_results = try ctx.exes.conv_stem_step.results(allocator); + defer conv_step_results.deinit(allocator); + + var enc_step_args = try ctx.exes.encoder_step.args(allocator); + defer enc_step_args.deinit(allocator); + var enc_step_results = try ctx.exes.encoder_step.results(allocator); + defer enc_step_results.deinit(allocator); + + var adp_step_args = try ctx.exes.adapter_step.args(allocator); + defer adp_step_args.deinit(allocator); + var adp_step_results = try ctx.exes.adapter_step.results(allocator); + defer adp_step_results.deinit(allocator); + + var decode_args = try ctx.exes.decoder_decode.args(allocator); + defer decode_args.deinit(allocator); + var decode_results = try ctx.exes.decoder_decode.results(allocator); + defer decode_results.deinit(allocator); + + var mel_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .channels = 128, .time = sp.mel_per_step }, .f32), .{}); + defer mel_step_output.deinit(); + + var conv_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); + defer conv_step_output.deinit(); + + var enc_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = sp.dsf, .d = enc_dim }, .bf16), .{}); + defer enc_step_output.deinit(); + + var adapter_step_output: zml.Buffer = try .uninitialized(io, platform, .init(.{ .s = 1, .d = config.dim }, .bf16), .{}); + defer adapter_step_output.deinit(); + + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); + + // Pre-allocate reusable stdin sample buffers + const raw_buf = try allocator.alloc(u8, new_audio_per_step * 2); + defer allocator.free(raw_buf); + const sample_buf = try allocator.alloc(f32, new_audio_per_step); + defer allocator.free(sample_buf); + + // Persistent on-device token index buffers + var enc_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, prompt_len) * sp.dsf, .u32); + defer enc_token_index.deinit(); + var dec_token_index: zml.Buffer = try .scalar(io, platform, @as(u32, prompt_len), .u32); + defer dec_token_index.deinit(); + + // Wave visualization + transcript accumulation + var stdout = std.Io.File.stdout().writer(io, &.{}); + var wave = terminalwave.State.init(.{ + .title = "Voxtral Realtime", + .sensitivity = 5.0, + .num_bars = 50, + .half_height = 20, + }, &stdout.interface); + errdefer wave.deinit(); + + var transcript: std.ArrayListUnmanaged(u8) = .empty; + defer transcript.deinit(allocator); + + // Set up scroll region: wave is fixed at top, transcript scrolls below + const title_rows: u16 = if (wave.config.show_title) 3 else 0; + const transcript_start_row: u16 = title_rows + wave.config.half_height + 4; + + // Render initial empty wave + separator, set scroll region + _ = wave.render(0); + wave.renderSeparator(); + wave.fmt(tesc.set_scroll_region_fmt, .{transcript_start_row}); + wave.fmt(tesc.move_cursor_fmt, .{ transcript_start_row, wave.config.padding_left + 1 }); + wave.put(tesc.clear_to_end); + wave.flush(); + + var last_transcript_len: usize = 0; + var num_generated: usize = 0; + while (true) { + const generated_token = generated_token_slice.items(u32)[0]; + num_generated += 1; + + if (generated_token != streaming_pad_token and (streaming_word_token == null or generated_token != streaming_word_token.?)) { + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try transcript.appendSlice(allocator, chunk); + } + } + + if (generated_token == eos_token) break; + + // Read new audio from stdin + readStdinSamplesInto(stdin_reader, raw_buf, sample_buf) catch |err| switch (err) { + error.EndOfStream => break, + else => return err, + }; + + // Save cursor position (in transcript scroll region) + wave.writer.writeAll(tesc.save_cursor) catch {}; + + // Render wave with current audio level (fixed area above scroll region) + const rms = terminalwave.computeRms(sample_buf); + _ = wave.render(rms); + wave.renderSeparator(); + + // Restore cursor and print only new transcript text + wave.put(tesc.restore_cursor); + if (transcript.items.len > last_transcript_len) { + wave.put(tesc.bright_white); + wave.put(transcript.items[last_transcript_len..]); + wave.put(tesc.reset); + last_transcript_len = transcript.items.len; + } + wave.flush(); + + // Shift history left, append new samples + @memmove(audio_buf[0..audio_overlap], audio_buf[new_audio_per_step..]); + @memcpy(audio_buf[audio_overlap..], sample_buf); + + const audio_slice: zml.Slice = .init( + .init(.{sp.chunk_audio}, .f32), + std.mem.sliceAsBytes(audio_buf), + ); + var audio_buffer: zml.Buffer = try .fromSliceOpts(io, platform, audio_slice, .{ .wait = false }); + defer audio_buffer.deinit(); + + mel_step_args.set(.{ ctx.buffers.mel_spectrum, audio_buffer }); + ctx.exes.mel_step.call(mel_step_args, &mel_step_results); + mel_step_results.fill(.{&mel_step_output}); + + // Conv stem step: mel + conv states → output + updated conv states + conv_step_args.set(.{ ctx.buffers.encoder, mel_step_output, ctx.buffers.conv_state }); + ctx.exes.conv_stem_step.call(conv_step_args, &conv_step_results); + conv_step_results.fill(.{ &conv_step_output, ctx.buffers.conv_state }); + + // Encoder step: conv stem output -> encoded chunk + enc_step_args.set(.{ ctx.buffers.encoder, conv_step_output, enc_token_index, ctx.buffers.enc_kv, ctx.buffers.enc_attention_metadata }); + ctx.exes.encoder_step.call(enc_step_args, &enc_step_results); + enc_step_results.fill(.{ &enc_step_output, ctx.buffers.enc_kv, &enc_token_index }); + + // Adapter step: encoded chunk -> 1 audio embedding + adp_step_args.set(.{ ctx.buffers.adapter, enc_step_output }); + ctx.exes.adapter_step.call(adp_step_args, &adp_step_results); + adp_step_results.fill(.{&adapter_step_output}); + + // Decoder decode: 1 token + 1 audio embedding -> next token + decode_args.set(.{ ctx.buffers.decoder, current_token_buffer, adapter_step_output, dec_token_index, ctx.buffers.dec_kv, ctx.buffers.t_cond, ctx.buffers.rng.*, ctx.buffers.dec_attention_metadata }); + ctx.exes.decoder_decode.call(decode_args, &decode_results); + decode_results.fill(.{ ¤t_token_buffer, ctx.buffers.dec_kv, ctx.buffers.rng, &dec_token_index }); + + try current_token_buffer.toSlice(io, generated_token_slice); + } + + // Reset scroll region and exit wave visualization (returns to main screen) + wave.deinit(); + + const decode_duration = decode_start.untilNow(io, .awake); + stdout.interface.print("\n" ++ tesc.bold ++ "Transcription:" ++ tesc.reset ++ "\n{s}\n\n", .{transcript.items}) catch {}; + log.info("Decode done. Generated {} tokens in {D}: {:.3}tok/s", .{ + num_generated, + stdx.fmt.fmtDuration(decode_duration), + stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(decode_duration, num_generated)), + }); +} + +pub fn runPipeline( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + config: Config, + tokenizer: *zml.tokenizer.Tokenizer, + prompt_tokens: []const u32, + sp: StreamParams, + exes: CompiledExes, + mel_spectrum_buffers: *zml.Bufferized(LogMelSpectrogram), + encoder_buffers: *zml.Bufferized(Encoder), + adapter_buffers: *zml.Bufferized(Adapter), + decoder_buffers: *zml.Bufferized(Decoder), + enc_kv_cache: KvCache, + dec_kv_cache: KvCache, + conv_state: Encoder.ConvState, + enc_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + dec_attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), +) !void { + log.info("Running inference pipeline...", .{}); + + var stdin_buff: [2560 * 4]u8 = undefined; + var stdin_reader = std.Io.File.stdin().reader(io, &stdin_buff); + const stdin_reader_interface = &stdin_reader.interface; + + // 0. Read initial audio from stdin and build padded buffer for prefill + // We read the full mel buffer's worth of audio (minus left_pad) so the STFT + // overlap region at the tail has real audio instead of silence. + const full_audio_len = sp.prefill_audio_len; + const n_prefill_stdin: usize = full_audio_len - sp.left_pad; + const prefill_raw_buf = try allocator.alloc(u8, n_prefill_stdin * 2); + defer allocator.free(prefill_raw_buf); + const stdin_audio = try allocator.alloc(f32, n_prefill_stdin); + defer allocator.free(stdin_audio); + try readStdinSamplesInto(stdin_reader_interface, prefill_raw_buf, stdin_audio); + + // Pad: [left_pad zeros | stdin audio] + const prefill_audio = try allocator.alloc(f32, full_audio_len); + defer allocator.free(prefill_audio); + @memset(prefill_audio[0..@as(usize, sp.left_pad)], 0); + @memcpy(prefill_audio[@as(usize, sp.left_pad)..], stdin_audio); + + // full_audio_len: 50160 | n_prefill_stdin: 9200 + + // 1. Init KV caches + t_cond + RNG + var enc_kv_buffers = try enc_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&enc_kv_buffers); + + var dec_kv_buffers = try dec_kv_cache.initBuffer(io, platform); + defer KvCache.deinitBuffer(&dec_kv_buffers); + + var conv_state_buffers = try conv_state.initBuffer(io, platform); + defer Encoder.ConvState.deinitBuffer(&conv_state_buffers); + + const t_cond_data = computeTimeEmbedding(allocator, @as(f32, @floatFromInt(sp.n_delay_tokens)), config.dim); + defer allocator.free(t_cond_data); + const t_cond_slice: zml.Slice = .init( + .init(.{config.dim}, .f32), + std.mem.sliceAsBytes(t_cond_data), + ); + var t_cond_buffer: zml.Buffer = try .fromSlice(io, platform, t_cond_slice); + defer t_cond_buffer.deinit(); + + var rng_buffers = try Tensor.Rng.initBuffer(platform, 0, io); + defer Tensor.Rng.deinitBuffer(&rng_buffers); + + // 2. Assemble pipeline context + var ctx = PipelineContext{ + .allocator = allocator, + .io = io, + .platform = platform, + .config = config, + .sp = sp, + .exes = exes, + .buffers = .{ + .mel_spectrum = mel_spectrum_buffers, + .encoder = encoder_buffers, + .adapter = adapter_buffers, + .decoder = decoder_buffers, + .enc_kv = &enc_kv_buffers, + .dec_kv = &dec_kv_buffers, + .conv_state = &conv_state_buffers, + .t_cond = t_cond_buffer, + .rng = &rng_buffers, + .enc_attention_metadata = enc_attention_metadata_buffers, + .dec_attention_metadata = dec_attention_metadata_buffers, + }, + }; + + // 3. Prefill (mel -> conv stem -> encoder -> adapter -> decoder) + var generated_token_slice: zml.Slice = try .alloc(allocator, .init(.{@as(u32, 1)}, .u32)); + defer generated_token_slice.free(allocator); + + try runPrefill(&ctx, prefill_audio, prompt_tokens, generated_token_slice); + + // 4. Generation loop — seed the overlap with the tail of the prefill audio + const audio_overlap = sp.audio_overlap; + try runGenerationLoop(&ctx, tokenizer, prefill_audio[full_audio_len - audio_overlap ..], stdin_reader_interface, generated_token_slice); +} + +fn readStdinSamplesInto(reader: *std.Io.Reader, raw_buf: []u8, samples: []f32) !void { + try reader.readSliceAll(raw_buf); + for (0..samples.len) |i| { + samples[i] = @as(f32, @floatFromInt(std.mem.bytesToValue(i16, raw_buf[i * 2 ..][0..2]))) / 32768.0; + } +} diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a21eec76..3412b3c5 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -461,7 +461,7 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 84781cb6..654620cc 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -4,8 +4,8 @@ load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): http_archive( name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", build_file = "//:third_party/flashattn/BUILD.bazel", ) diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index eac7dbd3..9028c386 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -61,7 +61,7 @@ def zls_completion(name, deps, **kwargs): main = name + ".runner", data = [ "@rules_zig//zig:resolved_toolchain", - "@@//third_party/zls:resolved_toolchain", + "@zml//third_party/zls:resolved_toolchain", name + ".build_runner", ], deps = [ diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 5ca5fde0..d83e8f1a 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -80,7 +80,7 @@ def _zls_toolchains_repos_impl(mctx): ), sha256 = _ZLS_VERSIONS[platform], build_file_content = """\ -load("@@//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") zls_toolchain(name = "toolchain", bin = "zls") """, ) @@ -98,7 +98,7 @@ zls_toolchains = module_extension( ) def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @@//third_party/zls:zls_toolchain.bzl""" + build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" for platform, config in _ZLS_PLATFORMS.items(): build_content += """ @@ -107,7 +107,7 @@ toolchain( exec_compatible_with = {compatible_with}, target_compatible_with = {compatible_with}, toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@@//third_party/zls:toolchain_type", + toolchain_type = "@zml//third_party/zls:toolchain_type", ) """.format( platform = platform, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 497bc834..0b730d0f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -22,6 +22,7 @@ zig_library( srcs = [ "attention/attention.zig", "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -41,6 +42,7 @@ zig_library( "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", ], main = "zml.zig", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..8b5b1442 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -101,7 +101,9 @@ pub const Metadata = union(Backend) { } }; -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { +pub const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { return switch (parameters) { .vanilla => b: { // Generate the attention mask. @@ -114,7 +116,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); break :b attn_output; }, - .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), - .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2, opts), + .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3, opts), }; } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8673bc4e..a6ee5b3c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,16 +7,22 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); } } -pub fn register(platform: *zml.Platform) !void { +pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); } } @@ -52,6 +58,13 @@ const Buffer = struct { flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), ); } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } }; fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { @@ -121,7 +134,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: *zml.Platform) !void { + pub fn register(platform: *const zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } @@ -239,7 +252,7 @@ pub const fa2 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { _ = parameters; // autofix stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); @@ -261,7 +274,7 @@ pub const fa2 = struct { const num_heads_k = k_.dim(.h); const head_size = q_.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); } @@ -281,8 +294,8 @@ pub const fa2 = struct { .{q.shape()}, .{ //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), + .is_causal = opts.is_causal, + .window_size_left = opts.sliding_window, .window_size_right = @as(i32, -1), .max_seqlen_q = max_seqlen_q, .max_seqlen_k = max_seqlen_k, @@ -422,8 +435,9 @@ pub const fa3 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: AttentionOptions) zml.Tensor { _ = parameters; // autofix + _ = opts; // autofix stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); // TODO(Corendos): replace with cumsum @@ -469,3 +483,957 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads, + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..deb63ec7 --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, + cuda_fa3, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.devices[0].pjrt_device; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, + + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), + }; +} diff --git a/zml/exe.zig b/zml/exe.zig index f5503d65..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,30 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } }; pub const Results = struct { diff --git a/zml/io.zig b/zml/io.zig index 8b8703cf..404843d2 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -120,6 +120,14 @@ pub const TensorStore = struct { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; diff --git a/zml/module.zig b/zml/module.zig index 31637852..a87866d9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -195,7 +195,7 @@ pub fn compile( const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); + // log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); const num_devices = sharding.num_partitions * sharding.num_replicas; const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); diff --git a/zml/nn.zig b/zml/nn.zig index cdc2b66b..4f75931f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,9 +27,10 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); + stdx.debug.assert(self.weight.dtype() == x.dtype(), "Linear expects input and weight to have the same dtype, got {} and {}", .{ x.dtype(), self.weight.dtype() }); + + var y = x.dot(self.weight, self.tag); - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; @@ -91,12 +92,13 @@ pub const LayerNorm = struct { } }; -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); } /// Center and scale by the variance. @@ -143,7 +145,7 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, freq_base: f32 = 10_000, - scaling: Scaling = .default, + scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. @@ -153,10 +155,14 @@ pub const RopeOpts = struct { /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { - default: void, - custom: []const f32, + default: Default, llama3: Llama3, yarn: Yarn, + linear: Scaling.Linear, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; pub const Llama3 = struct { factor: f32, @@ -164,43 +170,94 @@ pub const RopeOpts = struct { low_freq_factor: f32, original_max_position_embeddings: u32, truncate: bool = true, + rope_theta: f32 = 10000, }; pub const Yarn = struct { beta_fast: f32 = 32.0, beta_slow: f32 = 1.0, - factor: f32, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, + rope_theta: f32, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, + }; + + pub const Linear = struct { + factor: f32, + rope_theta: f32 = 10000, }; /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; // autofix + _ = options; // autofix + if (source == .null) return .{ .default = .{} }; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; + return .{ .default = .{} }; } } + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + pub fn attentionScaling(scaling: Scaling) f32 { return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, else => 1.0, }; } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } }; }; @@ -285,9 +342,8 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 @@ -317,7 +373,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; + const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; diff --git a/zml/ops.zig b/zml/ops.zig index 757949cd..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1063,3 +1063,10 @@ fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); +} diff --git a/zml/slice.zig b/zml/slice.zig index d9fb9956..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -1,9 +1,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; pub fn isBytes(comptime T: type) bool { @@ -25,6 +25,7 @@ pub const Slice = struct { pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); return if (type_info.pointer.is_const) .{ .inner_data = .{ .immutable = bytes }, .shape = shape } diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -287,10 +287,13 @@ pub fn testLayer( layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, - comptime name: []const u8, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), opts: CompareOpts, ) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -301,13 +304,14 @@ pub fn testLayer( index: usize = 0, }; + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(name ++ ".in"); + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -322,9 +326,10 @@ pub fn testLayer( const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; - const store_output = activation_store.withPrefix(name ++ ".out"); - const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -365,7 +370,7 @@ pub fn testLayer( try reader.interface.readSliceAll(expected_slice.data()); break :b expected_slice; }; - defer allocator.free(expected_slice.data()); + defer expected_slice.free(allocator); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -375,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); failed = true; continue; }, diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 4d576110..58444934 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -14,6 +14,7 @@ pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); @@ -43,6 +44,7 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This());